Skip to content
Python

added kandinsky 5 support

DeepBeepMeep3b652cf5ae76
diff --git a/defaults/k5_lite_i2v.json b/defaults/k5_lite_i2v.json
new file mode 100644
index 0000000..06ef594
--- /dev/null
+++ b/defaults/k5_lite_i2v.json
@@ -0,0 +1,16 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Lite I2V SFT 5s 2B",
+ "architecture": "k5_lite_i2v",
+ "description": "Kandinsky 5.0 Video Lite is a line-up of 2B models that generates high-quality HD videos from an input image and English or Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5lite_i2v_5s_bf16.safetensors"
+ ]
+ },
+ "prompt": "The subject slowly turns toward the camera, with subtle lighting changes and gentle motion.",
+ "num_inference_steps": 30,
+ "guidance_scale": 5.0,
+ "flow_shift": 10.0,
+ "video_length": 81
+}
diff --git a/defaults/k5_lite_t2v.json b/defaults/k5_lite_t2v.json
new file mode 100644
index 0000000..3dea66c
--- /dev/null
+++ b/defaults/k5_lite_t2v.json
@@ -0,0 +1,15 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Lite T2V SFT 5s 2B",
+ "architecture": "k5_lite_t2v",
+ "description": "Kandinsky 5.0 Video Lite is a line-up of 2B models that generates high-quality HD videos from English and Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5lite_t2v_sft_5s_bf16.safetensors"
+ ]
+ },
+ "num_inference_steps": 30,
+ "guidance_scale": 5.0,
+ "flow_shift": 10.0,
+ "video_length": 81
+}
diff --git a/defaults/k5_lite_t2v_10s_distil.json b/defaults/k5_lite_t2v_10s_distil.json
new file mode 100644
index 0000000..7a3aea9
--- /dev/null
+++ b/defaults/k5_lite_t2v_10s_distil.json
@@ -0,0 +1,20 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Lite T2V Distilled 10s 2B",
+ "architecture": "k5_lite_t2v",
+ "description": "Kandinsky 5.0 Video Lite is a line-up of 2B models that generates high-quality HD videos from English and Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5lite_t2v_distilled16steps_10s_bf16.safetensors"
+ ],
+ "k5_config_overrides": {
+ "metrics": {
+ "scheduler_scale": 10
+ }
+ }
+ },
+ "num_inference_steps": 16,
+ "guidance_scale": 1.0,
+ "flow_shift": 10.0,
+ "video_length": 161
+}
diff --git a/defaults/k5_lite_t2v_10s_distil_sparse.json b/defaults/k5_lite_t2v_10s_distil_sparse.json
new file mode 100644
index 0000000..160b8d4
--- /dev/null
+++ b/defaults/k5_lite_t2v_10s_distil_sparse.json
@@ -0,0 +1,66 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Lite T2V Distilled 10s Sparse 2B",
+ "architecture": "k5_lite_t2v",
+ "description": "Kandinsky 5.0 Video Lite is a line-up of 2B models that generates high-quality HD videos from English and Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "visible": false,
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5lite_t2v_distilled16steps_10s_bf16.safetensors"
+ ],
+ "resolutions": [
+ [
+ "1024x1024 (1:1)",
+ "1024x1024"
+ ],
+ [
+ "768x1024 (3:4)",
+ "768x1024"
+ ],
+ [
+ "1024x768 (4:3)",
+ "1024x768"
+ ],
+ [
+ "512x1024 (1:2)",
+ "512x1024"
+ ],
+ [
+ "1024x512 (2:1)",
+ "1024x512"
+ ],
+ [
+ "768x768 (1:1)",
+ "768x768"
+ ],
+ [
+ "768x512 (3:2)",
+ "768x512"
+ ],
+ [
+ "512x768 (2:3)",
+ "512x768"
+ ]
+ ],
+ "k5_config_overrides": {
+ "metrics": {
+ "scheduler_scale": 10
+ },
+ "model": {
+ "attention": {
+ "type": "nabla",
+ "P": 0.9,
+ "wT": 11,
+ "wW": 3,
+ "wH": 3,
+ "add_sta": true,
+ "method": "topcdf"
+ }
+ }
+ }
+ },
+ "num_inference_steps": 16,
+ "guidance_scale": 1.0,
+ "flow_shift": 10.0,
+ "video_length": 161
+}
diff --git a/defaults/k5_lite_t2v_5s_distil.json b/defaults/k5_lite_t2v_5s_distil.json
new file mode 100644
index 0000000..08d752b
--- /dev/null
+++ b/defaults/k5_lite_t2v_5s_distil.json
@@ -0,0 +1,20 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Lite T2V Distilled 5s 2B",
+ "architecture": "k5_lite_t2v",
+ "description": "Kandinsky 5.0 Video Lite is a line-up of 2B models that generates high-quality HD videos from English and Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5lite_t2v_distilled16steps_5s_bf16.safetensors"
+ ],
+ "k5_config_overrides": {
+ "metrics": {
+ "scheduler_scale": 5
+ }
+ }
+ },
+ "num_inference_steps": 16,
+ "guidance_scale": 1.0,
+ "flow_shift": 10.0,
+ "video_length": 81
+}
diff --git a/defaults/k5_pro_i2v.json b/defaults/k5_pro_i2v.json
new file mode 100644
index 0000000..12be9b9
--- /dev/null
+++ b/defaults/k5_pro_i2v.json
@@ -0,0 +1,17 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Pro I2V SFT 5s 19B",
+ "architecture": "k5_pro_i2v",
+ "description": "Kandinsky 5.0 Video Pro is a line-up of 19B models that generates high-quality HD videos from an input image and English or Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_i2v_sft_5s_bf16.safetensors",
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_i2v_sft_5s_quanto_bf16_int8.safetensors"
+ ]
+ },
+ "prompt": "The subject slowly turns toward the camera, with subtle lighting changes and gentle motion.",
+ "num_inference_steps": 30,
+ "guidance_scale": 5.0,
+ "flow_shift": 10.0,
+ "video_length": 81
+}
diff --git a/defaults/k5_pro_t2v.json b/defaults/k5_pro_t2v.json
new file mode 100644
index 0000000..9ef5ff0
--- /dev/null
+++ b/defaults/k5_pro_t2v.json
@@ -0,0 +1,16 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Pro T2V SFT 5s 19B",
+ "architecture": "k5_pro_t2v",
+ "description": "Kandinsky 5.0 Video Pro is a line-up of 19B models that generates high-quality HD videos from English and Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_t2v_sft_5s_bf16.safetensors",
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_t2v_sft_5s_quanto_bf16_int8.safetensors"
+ ]
+ },
+ "num_inference_steps": 30,
+ "guidance_scale": 5.0,
+ "flow_shift": 10.0,
+ "video_length": 81
+}
diff --git a/defaults/k5_pro_t2v_10s_sft.json b/defaults/k5_pro_t2v_10s_sft.json
new file mode 100644
index 0000000..e903926
--- /dev/null
+++ b/defaults/k5_pro_t2v_10s_sft.json
@@ -0,0 +1,18 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Pro T2V SFT 10s 19B",
+ "architecture": "k5_pro_t2v",
+ "description": "Kandinsky 5.0 Video Pro is a line-up of 19B models that generates high-quality HD videos from English and Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "visible": true,
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_t2v_sft_10s_bf16.safetensors",
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_t2v_sft_10s_quanto_bf16_int8.safetensors"
+ ],
+ "k5_config_overrides": {}
+ },
+ "num_inference_steps": 30,
+ "guidance_scale": 5.0,
+ "flow_shift": 10.0,
+ "video_length": 161
+}
diff --git a/defaults/k5_pro_t2v_10s_sft_sparse.json b/defaults/k5_pro_t2v_10s_sft_sparse.json
new file mode 100644
index 0000000..a1df970
--- /dev/null
+++ b/defaults/k5_pro_t2v_10s_sft_sparse.json
@@ -0,0 +1,64 @@
+{
+ "model": {
+ "name": "Kandinsky 5 Pro T2V SFT 10s Sparse 19B",
+ "architecture": "k5_pro_t2v",
+ "description": "Kandinsky 5.0 Video Pro is a line-up of 19B models that generates high-quality HD videos from English and Russian prompts with controllable camera motion.",
+ "group": "kandinsky5",
+ "visible": false,
+ "URLs": [
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_t2v_sft_10s_bf16.safetensors",
+ "https://huggingface.co/DeepBeepMeep/Kandinsky5/resolve/main/kandinsky5pro_t2v_sft_10s_quanto_bf16_int8.safetensors"
+ ],
+ "resolutions": [
+ [
+ "1024x1024 (1:1)",
+ "1024x1024"
+ ],
+ [
+ "768x1024 (3:4)",
+ "768x1024"
+ ],
+ [
+ "1024x768 (4:3)",
+ "1024x768"
+ ],
+ [
+ "512x1024 (1:2)",
+ "512x1024"
+ ],
+ [
+ "1024x512 (2:1)",
+ "1024x512"
+ ],
+ [
+ "768x768 (1:1)",
+ "768x768"
+ ],
+ [
+ "768x512 (3:2)",
+ "768x512"
+ ],
+ [
+ "512x768 (2:3)",
+ "512x768"
+ ]
+ ],
+ "k5_config_overrides": {
+ "model": {
+ "attention": {
+ "type": "nabla",
+ "method": "topcdf",
+ "P": 0.9,
+ "add_sta": true,
+ "wT": 11,
+ "wW": 3,
+ "wH": 3
+ }
+ }
+ }
+ },
+ "num_inference_steps": 30,
+ "guidance_scale": 5.0,
+ "flow_shift": 10.0,
+ "video_length": 161
+}
diff --git a/models/kandinsky5/__init__.py b/models/kandinsky5/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/models/kandinsky5/__init__.py
@@ -0,0 +1 @@
+
diff --git a/models/kandinsky5/configs/k5_lite_i2v.yaml b/models/kandinsky5/configs/k5_lite_i2v.yaml
new file mode 100644
index 0000000..2b4f550
--- /dev/null
+++ b/models/kandinsky5/configs/k5_lite_i2v.yaml
@@ -0,0 +1,46 @@
+metrics:
+ scheduler_scale: 10
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ time_dim: 512
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 1792
+ ff_dim: 7168
+ num_text_blocks: 2
+ num_visual_blocks: 32
+ axes_dims:
+ - 16
+ - 24
+ - 24
+ visual_cond: true
+ in_text_dim: 3584
+ in_text_dim2: 768
+ attention:
+ type: flash
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: "hunyuan"
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.90524, 0.89864, 0.95225, 0.95382, 1.04828, 1.04658, 1.07989, 1.07921, 1.04345, 1.04345, 1.03547, 1.03589, 1.03715, 1.03745, 1.02909, 1.02907, 1.03391, 1.03416, 1.02366, 1.02392, 1.02359, 1.02415, 1.02256, 1.02289, 1.02447, 1.02502, 1.01824, 1.01857, 1.02236, 1.02261, 1.01704, 1.01736, 1.01838, 1.01857, 1.01886, 1.01961, 1.01619, 1.0169, 1.01655, 1.01702, 1.01477, 1.01545, 1.01926, 1.01981, 1.01145, 1.01227, 1.01276, 1.01308, 1.01583, 1.01619, 1.01587, 1.01641, 1.01386, 1.01418, 1.01441, 1.01505, 1.01473, 1.01491, 1.01346, 1.01407, 1.01319, 1.0137, 1.01283, 1.01313, 1.01412, 1.01407, 1.01222, 1.0127, 1.0131, 1.01315, 1.01398, 1.0143, 1.01483, 1.01498, 1.01158, 1.01176, 1.01342, 1.01365, 1.00893, 1.0095, 1.01223, 1.01237, 1.01005, 1.01045, 1.00694, 1.00679, 1.00691, 1.00742, 0.99846, 0.99916, 0.98563, 0.98549, 0.97116, 0.97145, 0.93116, 0.93069, 0.83437, 0.83375]
diff --git a/models/kandinsky5/configs/k5_lite_i2v_5s_sft_sd.yaml b/models/kandinsky5/configs/k5_lite_i2v_5s_sft_sd.yaml
new file mode 100644
index 0000000..2b4f550
--- /dev/null
+++ b/models/kandinsky5/configs/k5_lite_i2v_5s_sft_sd.yaml
@@ -0,0 +1,46 @@
+metrics:
+ scheduler_scale: 10
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ time_dim: 512
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 1792
+ ff_dim: 7168
+ num_text_blocks: 2
+ num_visual_blocks: 32
+ axes_dims:
+ - 16
+ - 24
+ - 24
+ visual_cond: true
+ in_text_dim: 3584
+ in_text_dim2: 768
+ attention:
+ type: flash
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: "hunyuan"
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.90524, 0.89864, 0.95225, 0.95382, 1.04828, 1.04658, 1.07989, 1.07921, 1.04345, 1.04345, 1.03547, 1.03589, 1.03715, 1.03745, 1.02909, 1.02907, 1.03391, 1.03416, 1.02366, 1.02392, 1.02359, 1.02415, 1.02256, 1.02289, 1.02447, 1.02502, 1.01824, 1.01857, 1.02236, 1.02261, 1.01704, 1.01736, 1.01838, 1.01857, 1.01886, 1.01961, 1.01619, 1.0169, 1.01655, 1.01702, 1.01477, 1.01545, 1.01926, 1.01981, 1.01145, 1.01227, 1.01276, 1.01308, 1.01583, 1.01619, 1.01587, 1.01641, 1.01386, 1.01418, 1.01441, 1.01505, 1.01473, 1.01491, 1.01346, 1.01407, 1.01319, 1.0137, 1.01283, 1.01313, 1.01412, 1.01407, 1.01222, 1.0127, 1.0131, 1.01315, 1.01398, 1.0143, 1.01483, 1.01498, 1.01158, 1.01176, 1.01342, 1.01365, 1.00893, 1.0095, 1.01223, 1.01237, 1.01005, 1.01045, 1.00694, 1.00679, 1.00691, 1.00742, 0.99846, 0.99916, 0.98563, 0.98549, 0.97116, 0.97145, 0.93116, 0.93069, 0.83437, 0.83375]
diff --git a/models/kandinsky5/configs/k5_lite_t2v.yaml b/models/kandinsky5/configs/k5_lite_t2v.yaml
new file mode 100644
index 0000000..4fb8a64
--- /dev/null
+++ b/models/kandinsky5/configs/k5_lite_t2v.yaml
@@ -0,0 +1,45 @@
+metrics:
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ time_dim: 512
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 1792
+ ff_dim: 7168
+ num_text_blocks: 2
+ num_visual_blocks: 32
+ axes_dims:
+ - 16
+ - 24
+ - 24
+ visual_cond: true
+ in_text_dim: 3584
+ in_text_dim2: 768
+ attention:
+ type: flash
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: "hunyuan"
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.91607, 0.91507, 0.95254, 0.95349, 1.04876, 1.04937, 1.0842, 1.084, 1.04372, 1.04445, 1.03521, 1.03559, 1.03906, 1.03904, 1.03104, 1.03132, 1.03618, 1.03671, 1.02519, 1.02512, 1.02595, 1.02603, 1.02378, 1.02447, 1.02427, 1.02514, 1.01967, 1.01996, 1.02266, 1.02269, 1.01885, 1.01951, 1.01913, 1.01977, 1.01944, 1.02018, 1.01664, 1.01707, 1.01682, 1.01723, 1.0155, 1.01611, 1.01998, 1.02022, 1.01194, 1.01244, 1.01626, 1.01555, 1.01611, 1.01654, 1.01545, 1.01579, 1.01362, 1.01376, 1.01589, 1.01627, 1.01527, 1.01521, 1.01301, 1.01334, 1.01415, 1.01444, 1.0144, 1.01464, 1.01444, 1.01442, 1.01361, 1.01399, 1.01397, 1.01408, 1.01412, 1.01432, 1.01453, 1.01454, 1.01341, 1.01342, 1.01317, 1.01342, 1.01018, 1.01051, 1.01278, 1.0128, 1.01021, 1.01037, 1.00809, 1.00794, 1.00679, 1.00711, 0.99882, 0.99948, 0.98905, 0.98905, 0.9755, 0.97545, 0.93786, 0.93738, 0.84336, 0.84193]
diff --git a/models/kandinsky5/configs/k5_lite_t2v_10s_sft_sd.yaml b/models/kandinsky5/configs/k5_lite_t2v_10s_sft_sd.yaml
new file mode 100644
index 0000000..830def7
--- /dev/null
+++ b/models/kandinsky5/configs/k5_lite_t2v_10s_sft_sd.yaml
@@ -0,0 +1,52 @@
+metrics:
+ scheduler_scale: 10
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ time_dim: 512
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 1792
+ ff_dim: 7168
+ num_text_blocks: 2
+ num_visual_blocks: 32
+ axes_dims:
+ - 16
+ - 24
+ - 24
+ visual_cond: true
+ in_text_dim: 3584
+ in_text_dim2: 768
+ attention:
+ type: nabla
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ P: 0.9
+ wT: 11
+ wW: 3
+ wH: 3
+ add_sta: true
+ method: topcdf
+ vae:
+ name: "hunyuan"
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.92261, 0.92261, 0.95898, 0.95962, 1.04862, 1.04855, 1.0806, 1.08045, 1.04405, 1.0445, 1.03587, 1.03619, 1.03789, 1.03785, 1.03485, 1.03514, 1.03724, 1.03814, 1.02484, 1.02502, 1.02525, 1.02508, 1.02473, 1.02532, 1.02625, 1.02706, 1.0197, 1.02011, 1.02326, 1.02324, 1.02081, 1.02116, 1.01993, 1.02047, 1.01979, 1.0205, 1.01823, 1.01852, 1.01785, 1.01813, 1.01563, 1.01606, 1.02057, 1.02083, 1.01132, 1.01207, 1.02053, 1.01959, 1.01718, 1.01749, 1.01546, 1.01589, 1.01516, 1.01525, 1.01578, 1.01608, 1.01616, 1.01618, 1.01443, 1.01466, 1.01554, 1.01568, 1.01494, 1.01515, 1.01567, 1.01572, 1.01418, 1.01458, 1.01601, 1.01618, 1.01491, 1.01508, 1.0162, 1.01625, 1.01412, 1.01419, 1.01431, 1.01437, 1.0106, 1.0108, 1.01428, 1.01427, 1.01222, 1.01236, 1.00812, 1.00818, 1.00759, 1.00764, 1.001, 1.00119, 0.98798, 0.98819, 0.9727, 0.97279, 0.93234, 0.93213, 0.83781, 0.83746]
diff --git a/models/kandinsky5/configs/k5_lite_t2v_5s_sft_sd.yaml b/models/kandinsky5/configs/k5_lite_t2v_5s_sft_sd.yaml
new file mode 100644
index 0000000..4fb8a64
--- /dev/null
+++ b/models/kandinsky5/configs/k5_lite_t2v_5s_sft_sd.yaml
@@ -0,0 +1,45 @@
+metrics:
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ time_dim: 512
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 1792
+ ff_dim: 7168
+ num_text_blocks: 2
+ num_visual_blocks: 32
+ axes_dims:
+ - 16
+ - 24
+ - 24
+ visual_cond: true
+ in_text_dim: 3584
+ in_text_dim2: 768
+ attention:
+ type: flash
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: "hunyuan"
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.91607, 0.91507, 0.95254, 0.95349, 1.04876, 1.04937, 1.0842, 1.084, 1.04372, 1.04445, 1.03521, 1.03559, 1.03906, 1.03904, 1.03104, 1.03132, 1.03618, 1.03671, 1.02519, 1.02512, 1.02595, 1.02603, 1.02378, 1.02447, 1.02427, 1.02514, 1.01967, 1.01996, 1.02266, 1.02269, 1.01885, 1.01951, 1.01913, 1.01977, 1.01944, 1.02018, 1.01664, 1.01707, 1.01682, 1.01723, 1.0155, 1.01611, 1.01998, 1.02022, 1.01194, 1.01244, 1.01626, 1.01555, 1.01611, 1.01654, 1.01545, 1.01579, 1.01362, 1.01376, 1.01589, 1.01627, 1.01527, 1.01521, 1.01301, 1.01334, 1.01415, 1.01444, 1.0144, 1.01464, 1.01444, 1.01442, 1.01361, 1.01399, 1.01397, 1.01408, 1.01412, 1.01432, 1.01453, 1.01454, 1.01341, 1.01342, 1.01317, 1.01342, 1.01018, 1.01051, 1.01278, 1.0128, 1.01021, 1.01037, 1.00809, 1.00794, 1.00679, 1.00711, 0.99882, 0.99948, 0.98905, 0.98905, 0.9755, 0.97545, 0.93786, 0.93738, 0.84336, 0.84193]
diff --git a/models/kandinsky5/configs/k5_pro_i2v.yaml b/models/kandinsky5/configs/k5_pro_i2v.yaml
new file mode 100644
index 0000000..539b4ba
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_i2v.yaml
@@ -0,0 +1,47 @@
+metrics:
+ scheduler_scale: 10
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: flash
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.95295, 0.96037, 1.11843, 1.10903, 1.01421, 1.01204, 1.01811, 1.01562, 1.01231, 1.01056, 0.9972, 0.99662, 1.01979, 1.01757, 1.02381, 1.02207, 0.99479, 0.9951, 1.00294, 1.0024, 0.99773, 0.9984, 1.01817, 1.01605, 0.99913, 0.99944, 0.99218, 0.99299, 1.03239, 1.03021, 0.99568, 0.9951, 1.0124, 1.01201, 0.97933, 0.98116, 1.02511, 1.02369, 0.98456, 0.98444, 1.01573, 1.01525, 1.01743, 1.01688, 0.99326, 0.99416, 0.99642, 0.99641, 1.03137, 1.0265, 1.00513, 1.00922, 0.97789, 0.98044, 1.02419, 1.02194, 1.00083, 1.00147, 0.99584, 0.9971, 1.01021, 1.00855, 1.03171, 1.02869, 0.97483, 0.97822, 1.03725, 1.02688, 1.00433, 1.00196, 0.99005, 0.99085, 1.01196, 1.01015, 0.99815, 1.00002, 1.00673, 1.00522, 1.00589, 1.00883, 1.00424, 1.00184, 0.9863, 0.98711, 1.00569, 1.00432, 0.99096, 0.98978, 1.00107, 0.99868, 0.97743, 0.97855, 0.96446, 0.96312, 0.93896, 0.9413, 0.85115, 0.84801]
diff --git a/models/kandinsky5/configs/k5_pro_i2v_5s_sft_hd.yaml b/models/kandinsky5/configs/k5_pro_i2v_5s_sft_hd.yaml
new file mode 100644
index 0000000..913f0e3
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_i2v_5s_sft_hd.yaml
@@ -0,0 +1,53 @@
+metrics:
+ scheduler_scale: 5
+ scale_factor:
+ - 1.0
+ - 3.16
+ - 3.16
+ resolution: 1024
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: nabla
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ method: topcdf
+ P: 0.8
+ add_sta: true
+ wT: 11
+ wW: 3
+ wH: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [1.03563, 1.03368, 1.04423, 1.03794, 1.00481, 1.00407, 1.0317, 1.02925, 1.03814, 1.03604, 0.99118, 0.99214, 1.00768, 1.00728, 0.9999, 1.00075, 1.03272, 1.03035, 1.01254, 1.01205, 0.9801, 0.98287, 1.02185, 1.02018, 0.99365, 0.99368, 1.00572, 1.00619, 1.01411, 1.01344, 1.00445, 1.00438, 1.11448, 1.11342, 0.94186, 0.94065, 0.99309, 0.99454, 1.0088, 1.00838, 1.00741, 1.00808, 1.00698, 1.00605, 1.01266, 1.01218, 1.0106, 1.00968, 0.98969, 0.99204, 1.01784, 1.00727, 1.00304, 1.0025, 1.00326, 1.00309, 0.99478, 0.99558, 1.00362, 1.00373, 1.00801, 1.00714, 1.00184, 1.00202, 1.0099, 1.01111, 0.99854, 0.99918, 0.99393, 0.99442, 1.00183, 1.00176, 1.00648, 1.00565, 0.98875, 0.98903, 1.00561, 1.00376, 0.99138, 0.9926, 1.01195, 1.01037, 0.97298, 0.97494, 0.99551, 0.99309, 0.96925, 0.96974, 0.9741, 0.97415, 0.95861, 0.96225, 0.94539, 0.94383, 0.89835, 0.89666, 0.81038, 0.81322]
diff --git a/models/kandinsky5/configs/k5_pro_i2v_5s_sft_sd.yaml b/models/kandinsky5/configs/k5_pro_i2v_5s_sft_sd.yaml
new file mode 100644
index 0000000..539b4ba
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_i2v_5s_sft_sd.yaml
@@ -0,0 +1,47 @@
+metrics:
+ scheduler_scale: 10
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: flash
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.95295, 0.96037, 1.11843, 1.10903, 1.01421, 1.01204, 1.01811, 1.01562, 1.01231, 1.01056, 0.9972, 0.99662, 1.01979, 1.01757, 1.02381, 1.02207, 0.99479, 0.9951, 1.00294, 1.0024, 0.99773, 0.9984, 1.01817, 1.01605, 0.99913, 0.99944, 0.99218, 0.99299, 1.03239, 1.03021, 0.99568, 0.9951, 1.0124, 1.01201, 0.97933, 0.98116, 1.02511, 1.02369, 0.98456, 0.98444, 1.01573, 1.01525, 1.01743, 1.01688, 0.99326, 0.99416, 0.99642, 0.99641, 1.03137, 1.0265, 1.00513, 1.00922, 0.97789, 0.98044, 1.02419, 1.02194, 1.00083, 1.00147, 0.99584, 0.9971, 1.01021, 1.00855, 1.03171, 1.02869, 0.97483, 0.97822, 1.03725, 1.02688, 1.00433, 1.00196, 0.99005, 0.99085, 1.01196, 1.01015, 0.99815, 1.00002, 1.00673, 1.00522, 1.00589, 1.00883, 1.00424, 1.00184, 0.9863, 0.98711, 1.00569, 1.00432, 0.99096, 0.98978, 1.00107, 0.99868, 0.97743, 0.97855, 0.96446, 0.96312, 0.93896, 0.9413, 0.85115, 0.84801]
diff --git a/models/kandinsky5/configs/k5_pro_t2v.yaml b/models/kandinsky5/configs/k5_pro_t2v.yaml
new file mode 100644
index 0000000..d2ac250
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_t2v.yaml
@@ -0,0 +1,46 @@
+metrics:
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: flash
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [1.05108, 1.05459, 1.03217, 1.02625, 1.02084, 1.01846, 1.03457, 1.03179, 1.00932, 1.00985, 0.99931, 0.9993, 1.02284, 1.01996, 0.98916, 0.99098, 1.03845, 1.03547, 1.00028, 1.0004, 0.98184, 0.98364, 1.02553, 1.02362, 0.99988, 0.99932, 1.01271, 1.01285, 1.00014, 1.00077, 0.99271, 0.99318, 1.11646, 1.11503, 0.93333, 0.93057, 0.98949, 0.99107, 1.00834, 1.00772, 1.00786, 1.00798, 1.00357, 1.00307, 1.02427, 1.02217, 0.99526, 0.99595, 0.98479, 0.98713, 1.04114, 1.02792, 1.00598, 1.00493, 0.99466, 0.99516, 0.99617, 0.99626, 0.99749, 0.99835, 1.01384, 1.01141, 1.00413, 1.00386, 1.00521, 1.00832, 0.99833, 0.99848, 0.99242, 0.99254, 1.00186, 1.00105, 1.00468, 1.00341, 0.98685, 0.98733, 1.00656, 1.00466, 0.98858, 0.98928, 1.01212, 1.00985, 0.98267, 0.98204, 0.99826, 0.99613, 0.96953, 0.96927, 0.98073, 0.97957, 0.95653, 0.95842, 0.94036, 0.93953, 0.89943, 0.89771, 0.8026, 0.80511]
diff --git a/models/kandinsky5/configs/k5_pro_t2v_10s_sft_hd.yaml b/models/kandinsky5/configs/k5_pro_t2v_10s_sft_hd.yaml
new file mode 100644
index 0000000..75fd56f
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_t2v_10s_sft_hd.yaml
@@ -0,0 +1,52 @@
+metrics:
+ scale_factor:
+ - 1.0
+ - 3.16
+ - 3.16
+ resolution: 1024
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: nabla
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ method: topcdf
+ P: 0.7
+ add_sta: true
+ wT: 11
+ wW: 3
+ wH: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.98274, 0.99493, 1.03168, 1.02614, 1.01401, 1.01221, 1.03825, 1.03402, 1.01536, 1.01369, 0.99859, 0.99838, 1.0256, 1.02299, 1.01021, 1.00946, 1.01049, 1.01061, 0.99666, 0.99734, 0.99738, 0.99721, 1.02686, 1.02436, 0.99975, 1.00001, 1.00307, 1.00302, 1.0112, 1.01022, 0.98854, 0.99003, 1.10228, 1.10421, 0.94015, 0.93541, 0.99062, 0.99209, 1.00531, 1.00466, 1.0125, 1.01194, 0.99846, 0.99835, 1.01178, 1.01121, 1.0105, 1.00914, 0.98608, 0.988, 1.04434, 1.03336, 0.99672, 0.99642, 0.99961, 0.99924, 0.99378, 0.99426, 1.00752, 1.00722, 1.00349, 1.00256, 1.00613, 1.00526, 1.01463, 1.01604, 0.98702, 0.98944, 0.99258, 0.99203, 1.00309, 1.00248, 0.99763, 0.99656, 0.98566, 0.98628, 1.00527, 1.00374, 0.98961, 0.98984, 1.00967, 1.00762, 0.98694, 0.9865, 0.98838, 0.9866, 0.96498, 0.9649, 0.98126, 0.98001, 0.94053, 0.94297, 0.94064, 0.93973, 0.90176, 0.89994, 0.82025, 0.82369]
diff --git a/models/kandinsky5/configs/k5_pro_t2v_10s_sft_sd.yaml b/models/kandinsky5/configs/k5_pro_t2v_10s_sft_sd.yaml
new file mode 100644
index 0000000..03846af
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_t2v_10s_sft_sd.yaml
@@ -0,0 +1,52 @@
+metrics:
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: nabla
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ method: topcdf
+ P: 0.9
+ add_sta: true
+ wT: 11
+ wW: 3
+ wH: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [0.98023, 0.99247, 1.03089, 1.02514, 1.01309, 1.01089, 1.03909, 1.03481, 1.01614, 1.01475, 0.99935, 0.99898, 1.02636, 1.02427, 1.01073, 1.00992, 1.01095, 1.01107, 0.99695, 0.99739, 0.99787, 0.99773, 1.02732, 1.02448, 0.99997, 1.00002, 1.00324, 1.00319, 1.01161, 1.01049, 0.98849, 0.99007, 1.10221, 1.10528, 0.94084, 0.93506, 0.99063, 0.9922, 1.00556, 1.00493, 1.01289, 1.01203, 0.99886, 0.99879, 1.01212, 1.01146, 1.01083, 1.00942, 0.98618, 0.98824, 1.04545, 1.03401, 0.9971, 0.99662, 1.00007, 0.99974, 0.9939, 0.99448, 1.00796, 1.00782, 1.00393, 1.00306, 1.00658, 1.00564, 1.01497, 1.01626, 0.98722, 0.99015, 0.99306, 0.99243, 1.00389, 1.00315, 0.99839, 0.99737, 0.98623, 0.987, 1.00564, 1.00449, 0.99039, 0.99077, 1.01105, 1.00893, 0.98778, 0.98706, 0.98957, 0.98776, 0.96636, 0.96637, 0.98293, 0.98142, 0.94226, 0.9443, 0.94195, 0.94136, 0.90399, 0.902, 0.82158, 0.82574]
diff --git a/models/kandinsky5/configs/k5_pro_t2v_5s_sft_hd.yaml b/models/kandinsky5/configs/k5_pro_t2v_5s_sft_hd.yaml
new file mode 100644
index 0000000..c61f2d6
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_t2v_5s_sft_hd.yaml
@@ -0,0 +1,52 @@
+metrics:
+ scale_factor:
+ - 1.0
+ - 3.16
+ - 3.16
+ resolution: 1024
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: nabla
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ method: topcdf
+ P: 0.8
+ add_sta: true
+ wT: 11
+ wW: 3
+ wH: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [1.06133, 1.06278, 1.03873, 1.03381, 1.02178, 1.0193, 1.03491, 1.03217, 1.00996, 1.0107, 1.00116, 1.00166, 1.02416, 1.02113, 0.99041, 0.9923, 1.03965, 1.03658, 1.00088, 1.00101, 0.98248, 0.98433, 1.02611, 1.0244, 1.00062, 0.99987, 1.01328, 1.01354, 1.00058, 1.00124, 0.99323, 0.99369, 1.11828, 1.11693, 0.93251, 0.92965, 0.9899, 0.99161, 1.00862, 1.00802, 1.00781, 1.00803, 1.00378, 1.003, 1.02443, 1.02223, 0.99546, 0.99611, 0.98498, 0.98758, 1.04163, 1.02739, 1.00603, 1.00499, 0.99486, 0.99531, 0.99648, 0.99687, 0.99773, 0.99845, 1.01416, 1.01168, 1.00427, 1.00404, 1.00567, 1.00861, 0.99845, 0.99858, 0.99266, 0.99283, 1.00193, 1.00096, 1.00457, 1.00317, 0.98699, 0.98747, 1.00696, 1.00524, 0.98907, 0.98964, 1.01238, 1.00992, 0.98305, 0.98243, 0.99844, 0.99614, 0.96994, 0.96961, 0.98165, 0.98019, 0.95794, 0.95881, 0.94205, 0.94027, 0.90171, 0.89945, 0.80597, 0.80726]
diff --git a/models/kandinsky5/configs/k5_pro_t2v_5s_sft_sd.yaml b/models/kandinsky5/configs/k5_pro_t2v_5s_sft_sd.yaml
new file mode 100644
index 0000000..d2ac250
--- /dev/null
+++ b/models/kandinsky5/configs/k5_pro_t2v_5s_sft_sd.yaml
@@ -0,0 +1,46 @@
+metrics:
+ scale_factor:
+ - 1.0
+ - 2.0
+ - 2.0
+ resolution: 512
+model:
+ num_steps: 50
+ guidance_weight: 5.0
+ dit_params:
+ in_visual_dim: 16
+ out_visual_dim: 16
+ in_text_dim: 3584
+ in_text_dim2: 768
+ time_dim: 1024
+ patch_size:
+ - 1
+ - 2
+ - 2
+ model_dim: 4096
+ ff_dim: 16384
+ num_text_blocks: 4
+ num_visual_blocks: 60
+ axes_dims:
+ - 32
+ - 48
+ - 48
+ visual_cond: true
+ attention:
+ type: flash
+ chunk: false
+ causal: false
+ local: false
+ glob: false
+ window: 3
+ vae:
+ name: hunyuan
+ text_embedder:
+ qwen:
+ emb_size: 3584
+ max_length: 256
+ clip:
+ emb_size: 768
+ max_length: 77
+magcache:
+ mag_ratios: [1.05108, 1.05459, 1.03217, 1.02625, 1.02084, 1.01846, 1.03457, 1.03179, 1.00932, 1.00985, 0.99931, 0.9993, 1.02284, 1.01996, 0.98916, 0.99098, 1.03845, 1.03547, 1.00028, 1.0004, 0.98184, 0.98364, 1.02553, 1.02362, 0.99988, 0.99932, 1.01271, 1.01285, 1.00014, 1.00077, 0.99271, 0.99318, 1.11646, 1.11503, 0.93333, 0.93057, 0.98949, 0.99107, 1.00834, 1.00772, 1.00786, 1.00798, 1.00357, 1.00307, 1.02427, 1.02217, 0.99526, 0.99595, 0.98479, 0.98713, 1.04114, 1.02792, 1.00598, 1.00493, 0.99466, 0.99516, 0.99617, 0.99626, 0.99749, 0.99835, 1.01384, 1.01141, 1.00413, 1.00386, 1.00521, 1.00832, 0.99833, 0.99848, 0.99242, 0.99254, 1.00186, 1.00105, 1.00468, 1.00341, 0.98685, 0.98733, 1.00656, 1.00466, 0.98858, 0.98928, 1.01212, 1.00985, 0.98267, 0.98204, 0.99826, 0.99613, 0.96953, 0.96927, 0.98073, 0.97957, 0.95653, 0.95842, 0.94036, 0.93953, 0.89943, 0.89771, 0.8026, 0.80511]
diff --git a/models/kandinsky5/kandinsky/__init__.py b/models/kandinsky5/kandinsky/__init__.py
new file mode 100644
index 0000000..3eea7da
--- /dev/null
+++ b/models/kandinsky5/kandinsky/__init__.py
@@ -0,0 +1 @@
+from .utils import get_T2V_pipeline, get_I2V_pipeline, get_T2I_pipeline, get_I2I_pipeline
diff --git a/models/kandinsky5/kandinsky/generation_utils.py b/models/kandinsky5/kandinsky/generation_utils.py
new file mode 100644
index 0000000..3d152bb
--- /dev/null
+++ b/models/kandinsky5/kandinsky/generation_utils.py
@@ -0,0 +1,597 @@
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "False"
+
+import torch
+from tqdm import tqdm
+
+from .models.utils import fast_sta_nabla
+import torchvision.transforms.functional as F
+
+
+def get_sparse_params(conf, batch_embeds, device):
+ assert conf.model.dit_params.patch_size[0] == 1
+ T, H, W, _ = batch_embeds["visual"].shape
+ T, H, W = (
+ T // conf.model.dit_params.patch_size[0],
+ H // conf.model.dit_params.patch_size[1],
+ W // conf.model.dit_params.patch_size[2],
+ )
+ if conf.model.attention.type == "nabla":
+ sta_mask = fast_sta_nabla(T, H // 8, W // 8, conf.model.attention.wT,
+ conf.model.attention.wH, conf.model.attention.wW, device=device)
+ sparse_params = {
+ "sta_mask": sta_mask.unsqueeze_(0).unsqueeze_(0),
+ "attention_type": conf.model.attention.type,
+ "to_fractal": True,
+ "P": conf.model.attention.P,
+ "wT": conf.model.attention.wT,
+ "wW": conf.model.attention.wW,
+ "wH": conf.model.attention.wH,
+ "add_sta": conf.model.attention.add_sta,
+ "visual_shape": (T, H, W),
+ "method": getattr(conf.model.attention, "method", "topcdf"),
+ }
+ else:
+ sparse_params = None
+
+ return sparse_params
+
+def adaptive_mean_std_normalization(source, reference):
+ source_mean = source.mean(dim=(1,2,3),keepdim=True)
+ source_std = source.std(dim=(1,2,3),keepdim=True)
+ #magic constants - limit changes in latents
+ clump_mean_low = 0.05
+ clump_mean_high = 0.1
+ clump_std_low = 0.1
+ clump_std_high = 0.25
+
+ reference_mean = torch.clamp(reference.mean(), source_mean - clump_mean_low, source_mean + clump_mean_high)
+ reference_std = torch.clamp(reference.std(), source_std - clump_std_low, source_std + clump_std_high)
+
+ # normalization
+ normalized = (source - source_mean) / source_std
+ normalized = normalized * reference_std + reference_mean
+
+ return normalized
+
+def normalize_first_frame(latents, reference_frames=5, clump_values=False):
+ latents_copy = latents.clone()
+ samples = latents_copy
+
+ if samples.shape[0] <= 1:
+ return (latents, "Only one frame, no normalization needed")
+ nFr = 4
+ first_frames = samples[:nFr]
+ reference_frames_data = samples[nFr:nFr+min(reference_frames, samples.shape[0]-1)]
+
+ # print("First frame stats - Mean:", first_frames.mean(dim=(1,2,3)), "Std: ", first_frames.std(dim=(1,2,3)))
+ # print(f"Reference frames stats - Mean: {reference_frames_data.mean().item():.4f}, Std: {reference_frames_data.std().item():.4f}")
+
+ normalized_first = adaptive_mean_std_normalization(first_frames, reference_frames_data)
+ if clump_values:
+ min_val = reference_frames_data.min()
+ max_val = reference_frames_data.max()
+ normalized_first = torch.clamp(normalized_first, min_val, max_val)
+
+ samples[:nFr] = normalized_first
+
+ return samples
+
+def get_velocity(
+ dit,
+ x,
+ t,
+ text_embeds,
+ null_text_embeds,
+ visual_rope_pos,
+ text_rope_pos,
+ null_text_rope_pos,
+ guidance_weight,
+ conf,
+ sparse_params=None,
+ attention_mask=None,
+ null_attention_mask=None,
+ joint_pass=False,
+):
+ with torch._dynamo.utils.disable_cache_limit():
+ if joint_pass and abs(guidance_weight - 1.0) > 1e-6:
+ outputs = dit(
+ [x, x],
+ [text_embeds["text_embeds"], null_text_embeds["text_embeds"]],
+ [text_embeds["pooled_embed"], null_text_embeds["pooled_embed"]],
+ t * 1000,
+ [visual_rope_pos, visual_rope_pos],
+ [text_rope_pos, null_text_rope_pos],
+ scale_factor=[conf.metrics.scale_factor, conf.metrics.scale_factor],
+ sparse_params=[sparse_params, sparse_params],
+ attention_mask=[attention_mask, null_attention_mask],
+ )
+ if outputs is None:
+ return None
+ pred_velocity, uncond_pred_velocity = outputs
+ pred_velocity = uncond_pred_velocity + guidance_weight * (
+ pred_velocity - uncond_pred_velocity
+ )
+ else:
+ pred_velocity = dit(
+ x,
+ text_embeds["text_embeds"],
+ text_embeds["pooled_embed"],
+ t * 1000,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor=conf.metrics.scale_factor,
+ sparse_params=sparse_params,
+ attention_mask=attention_mask,
+ )
+ if pred_velocity is None:
+ return None
+ if abs(guidance_weight - 1.0) > 1e-6:
+ uncond_pred_velocity = dit(
+ x,
+ null_text_embeds["text_embeds"],
+ null_text_embeds["pooled_embed"],
+ t * 1000,
+ visual_rope_pos,
+ null_text_rope_pos,
+ scale_factor=conf.metrics.scale_factor,
+ sparse_params=sparse_params,
+ attention_mask=null_attention_mask,
+ )
+ if uncond_pred_velocity is None:
+ return None
+ pred_velocity = uncond_pred_velocity + guidance_weight * (
+ pred_velocity - uncond_pred_velocity
+ )
+ return pred_velocity
+
+
+def generate(
+ model,
+ device,
+ img,
+ num_steps,
+ text_embeds,
+ null_text_embeds,
+ visual_rope_pos,
+ text_rope_pos,
+ null_text_rope_pos,
+ guidance_weight,
+ scheduler_scale,
+ first_frames,
+ conf,
+ progress=False,
+ seed=6554,
+ attention_mask=None,
+ null_attention_mask=None,
+ callback=None,
+ interrupt_check=None,
+ joint_pass=False,
+):
+ sparse_params = get_sparse_params(conf, {"visual": img}, device)
+ timesteps = torch.linspace(1, 0, num_steps + 1, device=device)
+ timesteps = scheduler_scale * timesteps / (1 + (scheduler_scale - 1) * timesteps)
+
+ if callback is not None:
+ callback(-1, None, True, override_num_inference_steps=num_steps)
+
+ step_iter = zip(timesteps[:-1], torch.diff(timesteps))
+ if progress:
+ step_iter = tqdm(step_iter, total=num_steps)
+
+ for step_idx, (timestep, timestep_diff) in enumerate(step_iter):
+ if interrupt_check is not None and interrupt_check():
+ return None
+ time = timestep.unsqueeze(0)
+ if model.visual_cond:
+ visual_cond = torch.zeros_like(img)
+ visual_cond_mask = torch.zeros(
+ [*img.shape[:-1], 1], dtype=img.dtype, device=img.device
+ )
+ if first_frames is not None:
+ first_frames = first_frames.to(device=visual_cond.device, dtype=visual_cond.dtype)
+ img[:1] = first_frames
+ visual_cond_mask[:1] = 1
+ model_input = torch.cat([img, visual_cond, visual_cond_mask], dim=-1)
+ else:
+ model_input = img
+ pred_velocity = get_velocity(
+ model,
+ model_input,
+ time,
+ text_embeds,
+ null_text_embeds,
+ visual_rope_pos,
+ text_rope_pos,
+ null_text_rope_pos,
+ guidance_weight,
+ conf,
+ sparse_params=sparse_params,
+ attention_mask=attention_mask,
+ null_attention_mask=null_attention_mask,
+ joint_pass=joint_pass,
+ )
+ if pred_velocity is None:
+ return None
+ img[..., :pred_velocity.shape[-1]] += timestep_diff * pred_velocity
+ if callback is not None:
+ latents_preview = None
+ if visual_rope_pos is not None and len(visual_rope_pos) > 0:
+ duration = int(visual_rope_pos[0].numel())
+ if duration > 0 and img.shape[0] % duration == 0:
+ batch = img.shape[0] // duration
+ latents_preview = (
+ img.reshape(batch, duration, img.shape[1], img.shape[2], img.shape[3])
+ .permute(0, 4, 1, 2, 3)
+ .detach()
+ )[0]
+ callback(step_idx, latents_preview, False)
+ # NOTE: remove extra channels that can be added in Image Editing (I2I)
+ return img[..., :pred_velocity.shape[-1]]
+
+
+def resize_video(video, visual_size):
+ height, width = video.shape[-2:]
+ nearest_height, nearest_width = visual_size
+
+ scale_factor = min(height / nearest_height, width / nearest_width)
+ video = F.resize(video, (int(height / scale_factor), int(width / scale_factor)))
+
+ height, width = video.shape[-2:]
+ video = F.crop(
+ video,
+ (height - nearest_height) // 2,
+ (width - nearest_width) // 2,
+ nearest_height,
+ nearest_width,
+ )
+ return video
+
+
+def encode_video(data, vae, image_vae): # batch, channels, time, h, w
+ if image_vae:
+ assert data.shape[2] == 1
+ data = vae.encode(data[:, :, 0]).latent_dist.sample()[:, :, None]
+ else:
+ data = vae.encode(data)[0]
+ data *= vae.config.scaling_factor
+ return data.permute(0, 2, 3, 4, 1) # batch, time, h, w, channels
+
+
+def generate_sample(
+ shape,
+ caption,
+ dit,
+ vae,
+ conf,
+ text_embedder,
+ num_steps=25,
+ guidance_weight=5.0,
+ scheduler_scale=1,
+ negative_caption="",
+ seed=6554,
+ device="cuda",
+ vae_device="cuda",
+ text_embedder_device="cuda",
+ progress=True,
+ joint_pass=False,
+ callback=None,
+ interrupt_check=None,
+):
+ bs, duration, height, width, dim = shape
+
+ g = torch.Generator(device="cuda")
+ g.manual_seed(seed)
+ img = torch.randn(bs * duration, height, width, dim, device=device, generator=g, dtype=torch.bfloat16)
+
+ if duration == 1:
+ type_of_content = "image"
+ else:
+ type_of_content = "video"
+
+ with torch.no_grad():
+ bs_text_embed, text_cu_seqlens, attention_mask = text_embedder.encode(
+ [caption], type_of_content=type_of_content
+ )
+ bs_null_text_embed, null_text_cu_seqlens, null_attention_mask = text_embedder.encode(
+ [negative_caption], type_of_content=type_of_content
+ )
+
+ for key in bs_text_embed:
+ bs_text_embed[key] = bs_text_embed[key].to(device=device)
+ bs_null_text_embed[key] = bs_null_text_embed[key].to(device=device)
+ if attention_mask is not None:
+ attention_mask = attention_mask.to(device=device)
+ if null_attention_mask is not None:
+ null_attention_mask = null_attention_mask.to(device=device)
+ text_cu_seqlens = text_cu_seqlens.to(device=device)[-1].item()
+ null_text_cu_seqlens = null_text_cu_seqlens.to(device=device)[-1].item()
+
+ visual_rope_pos = [
+ torch.arange(duration),
+ torch.arange(shape[-3] // conf.model.dit_params.patch_size[1]),
+ torch.arange(shape[-2] // conf.model.dit_params.patch_size[2]),
+ ]
+ text_rope_pos = torch.arange(text_cu_seqlens)
+ null_text_rope_pos = torch.arange(null_text_cu_seqlens)
+
+ with torch.no_grad():
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
+ latent_visual = generate(
+ dit,
+ device,
+ img,
+ num_steps,
+ bs_text_embed,
+ bs_null_text_embed,
+ visual_rope_pos,
+ text_rope_pos,
+ null_text_rope_pos,
+ guidance_weight,
+ scheduler_scale,
+ None,
+ conf,
+ seed=seed,
+ progress=progress,
+ attention_mask=attention_mask,
+ null_attention_mask=null_attention_mask,
+ callback=callback,
+ interrupt_check=interrupt_check,
+ joint_pass=joint_pass,
+ )
+
+ if latent_visual is None:
+ return None
+
+
+ with torch.no_grad():
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
+ images = latent_visual.reshape(
+ bs,
+ -1,
+ latent_visual.shape[-3],
+ latent_visual.shape[-2],
+ latent_visual.shape[-1],
+ )
+ images = images.to(device=vae_device)
+ images = (images / vae.config.scaling_factor).permute(0, 4, 1, 2, 3)
+ images = vae.decode(images).sample
+ images = ((images.clamp(-1.0, 1.0) + 1.0) * 127.5).to(torch.uint8)
+
+
+ return images
+
+def generate_sample_ti2i(
+ shape,
+ caption,
+ dit,
+ vae,
+ conf,
+ text_embedder,
+ num_steps=25,
+ guidance_weight=5.0,
+ scheduler_scale=1,
+ negative_caption="",
+ seed=6554,
+ device="cuda",
+ vae_device="cuda",
+ text_embedder_device="cuda",
+ progress=True,
+ image_vae=False,
+ image=None,
+ joint_pass=False,
+ callback=None,
+ interrupt_check=None
+):
+ bs, duration, height, width, dim = shape
+
+ g = torch.Generator(device="cuda")
+ g.manual_seed(seed)
+ img = torch.randn(bs * duration, height, width, dim, device=device, generator=g, dtype=torch.bfloat16)
+
+ if duration == 1:
+ if image is None:
+ type_of_content = "image"
+ else:
+ type_of_content = 'image_edit'
+ else:
+ type_of_content = "video"
+
+
+ if image is not None:
+ image = [resize_video(image, (height * 8, width * 8))]
+
+ if dit.instruct_type == 'channel':
+ if image is not None:
+ edit_latent = [(i.to(device=vae_device, dtype=torch.bfloat16) / 127.5 - 1.0) for i in image]
+ edit_latent = torch.cat([encode_video(i[:,:,None], vae, image_vae).squeeze(0) for i in edit_latent], 0)
+ edit_latent = torch.cat([edit_latent, torch.ones_like(img[...,:1])],-1)
+ else:
+ edit_latent = torch.cat([torch.zeros_like(img), torch.zeros_like(img[...,:1])],-1)
+ img = torch.cat([img, edit_latent],dim=-1)
+
+ with torch.no_grad():
+ bs_text_embed, text_cu_seqlens, attention_mask = text_embedder.encode(
+ [caption], type_of_content=type_of_content, images=image
+ )
+ bs_null_text_embed, null_text_cu_seqlens, null_attention_mask = text_embedder.encode(
+ [negative_caption], type_of_content=type_of_content, images=image
+ )
+
+ for key in bs_text_embed:
+ bs_text_embed[key] = bs_text_embed[key].to(device=device,dtype=torch.bfloat16)
+ bs_null_text_embed[key] = bs_null_text_embed[key].to(device=device,dtype=torch.bfloat16)
+ if attention_mask is not None:
+ attention_mask = attention_mask.to(device=device)
+ if null_attention_mask is not None:
+ null_attention_mask = null_attention_mask.to(device=device)
+ text_cu_seqlens = text_cu_seqlens.to(device=device)[-1].item()
+ null_text_cu_seqlens = null_text_cu_seqlens.to(device=device)[-1].item()
+
+ visual_rope_pos = [
+ torch.arange(duration),
+ torch.arange(shape[-3] // conf.model.dit_params.patch_size[1]),
+ torch.arange(shape[-2] // conf.model.dit_params.patch_size[2]),
+ ]
+ text_rope_pos = torch.arange(text_cu_seqlens)
+ null_text_rope_pos = torch.arange(null_text_cu_seqlens)
+
+ with torch.no_grad():
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
+ latent_visual = generate(
+ dit,
+ device,
+ img,
+ num_steps,
+ bs_text_embed,
+ bs_null_text_embed,
+ visual_rope_pos,
+ text_rope_pos,
+ null_text_rope_pos,
+ guidance_weight,
+ scheduler_scale,
+ None,
+ conf,
+ seed=seed,
+ progress=progress,
+ attention_mask=attention_mask,
+ null_attention_mask=null_attention_mask,
+ callback=callback,
+ interrupt_check=interrupt_check,
+ joint_pass=joint_pass,
+ )
+
+ if latent_visual is None:
+ return None
+
+
+ with torch.no_grad():
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
+ images = latent_visual.reshape(
+ bs,
+ -1,
+ latent_visual.shape[-3],
+ latent_visual.shape[-2],
+ latent_visual.shape[-1],
+ )
+ images = images.to(device=vae_device)
+ images = (images / vae.config.scaling_factor).permute(0, 4, 1, 2, 3)
+ if image_vae:
+ images = images[:,:,0]
+ images = vae.decode(images).sample
+ images = ((images.clamp(-1.0, 1.0) + 1.0) * 127.5).to(torch.uint8)
+
+
+ return images
+
+def generate_sample_i2v(
+ shape,
+ caption,
+ dit,
+ vae,
+ conf,
+ text_embedder,
+ images,
+ num_steps=50,
+ guidance_weight=5.0,
+ scheduler_scale=1,
+ negative_caption="",
+ seed=6554,
+ device="cuda",
+ vae_device="cuda",
+ progress=True,
+ callback=None,
+ joint_pass=False,
+ interrupt_check=None
+):
+ text_embedder.embedder.mode = "i2v"
+ bs, duration, height, width, dim = shape
+
+ g = torch.Generator(device="cuda")
+ g.manual_seed(seed)
+ img = torch.randn(bs * duration, height, width, dim, device=device, generator=g, dtype=torch.bfloat16)
+
+ if duration == 1:
+ type_of_content = "image"
+ else:
+ type_of_content = "video"
+
+ with torch.no_grad():
+ bs_text_embed, text_cu_seqlens, attention_mask = text_embedder.encode(
+ [caption], type_of_content=type_of_content
+ )
+ bs_null_text_embed, null_text_cu_seqlens, null_attention_mask = text_embedder.encode(
+ [negative_caption], type_of_content=type_of_content
+ )
+
+ for key in bs_text_embed:
+ bs_text_embed[key] = bs_text_embed[key].to(device=device)
+ bs_null_text_embed[key] = bs_null_text_embed[key].to(device=device)
+ if attention_mask is not None:
+ attention_mask = attention_mask.to(device=device)
+ if null_attention_mask is not None:
+ null_attention_mask = null_attention_mask.to(device=device)
+ text_cu_seqlens = text_cu_seqlens.to(device=device)[-1].item()
+ null_text_cu_seqlens = null_text_cu_seqlens.to(device=device)[-1].item()
+
+ visual_rope_pos = [
+ torch.arange(duration),
+ torch.arange(shape[-3] // conf.model.dit_params.patch_size[1]),
+ torch.arange(shape[-2] // conf.model.dit_params.patch_size[2]),
+ ]
+ text_rope_pos = torch.arange(text_cu_seqlens)
+ null_text_rope_pos = torch.arange(null_text_cu_seqlens)
+
+ first_frames = images
+
+ with torch.no_grad():
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
+ latent_visual = generate(
+ dit,
+ device,
+ img,
+ num_steps,
+ bs_text_embed,
+ bs_null_text_embed,
+ visual_rope_pos,
+ text_rope_pos,
+ null_text_rope_pos,
+ guidance_weight,
+ scheduler_scale,
+ first_frames,
+ conf,
+ seed=seed,
+ progress=progress,
+ attention_mask=attention_mask,
+ null_attention_mask=null_attention_mask,
+ callback=callback,
+ interrupt_check=interrupt_check,
+ joint_pass=joint_pass,
+ )
+ if latent_visual is None:
+ return None
+
+ if images is not None:
+ images = images.to(device=latent_visual.device, dtype=latent_visual.dtype)
+ latent_visual[:1] = images
+ latent_visual = normalize_first_frame(latent_visual)
+
+
+ with torch.no_grad():
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
+ images = latent_visual.reshape(
+ bs,
+ -1,
+ latent_visual.shape[-3],
+ latent_visual.shape[-2],
+ latent_visual.shape[-1],
+ )
+ images = images.to(device=vae_device)
+ images = (images / vae.config.scaling_factor).permute(0, 4, 1, 2, 3)
+ images = vae.decode(images).sample
+ images = ((images.clamp(-1.0, 1.0) + 1.0) * 127.5).to(torch.uint8)
+
+
+ return images
diff --git a/models/kandinsky5/kandinsky/i2i_pipeline.py b/models/kandinsky5/kandinsky/i2i_pipeline.py
new file mode 100644
index 0000000..928d60f
--- /dev/null
+++ b/models/kandinsky5/kandinsky/i2i_pipeline.py
@@ -0,0 +1,65 @@
+from typing import Optional, Union
+
+import torch
+from PIL import Image
+
+from .pipeline import Kandinsky5Pipeline
+
+
+class Kandinsky5I2IPipeline(Kandinsky5Pipeline):
+ RESOLUTIONS = Kandinsky5Pipeline.IMAGE_RESOLUTIONS
+
+ def __init__(
+ self,
+ device_map: Union[str, torch.device, dict],
+ dit,
+ text_embedder,
+ vae,
+ resolution: int = 1024,
+ conf=None,
+ ):
+ if resolution not in [1024]:
+ raise ValueError("Resolution can be only 1024")
+ super().__init__(
+ mode="i2i",
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ resolution=resolution,
+ conf=conf,
+ )
+
+ def __call__(
+ self,
+ text: str,
+ width: Optional[int] = None,
+ height: Optional[int] = None,
+ seed: int = None,
+ num_steps: Optional[int] = None,
+ guidance_weight: Optional[float] = None,
+ scheduler_scale: float = 3.0,
+ negative_caption: str = "",
+ expand_prompts: bool = True,
+ save_path: str = None,
+ progress: bool = True,
+ image: Optional[Union[Image.Image, str]] = None,
+ joint_pass: bool = False,
+ ):
+ return super().__call__(
+ text=text,
+ image=image,
+ time_length=0,
+ width=width,
+ height=height,
+ frame_num=1,
+ seed=seed,
+ num_steps=num_steps,
+ guidance_weight=guidance_weight,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ expand_prompts=expand_prompts,
+ save_path=save_path,
+ progress=progress,
+ joint_pass=joint_pass,
+ )
diff --git a/models/kandinsky5/kandinsky/i2v_pipeline.py b/models/kandinsky5/kandinsky/i2v_pipeline.py
new file mode 100644
index 0000000..70b88d4
--- /dev/null
+++ b/models/kandinsky5/kandinsky/i2v_pipeline.py
@@ -0,0 +1,63 @@
+from typing import Optional, Union
+
+import torch
+from PIL import Image
+
+from .pipeline import Kandinsky5Pipeline
+
+
+class Kandinsky5I2VPipeline(Kandinsky5Pipeline):
+ def __init__(
+ self,
+ device_map: Union[str, torch.device, dict],
+ dit,
+ text_embedder,
+ vae,
+ conf=None,
+ ):
+ super().__init__(
+ mode="i2v",
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ resolution=conf.metrics.resolution,
+ conf=conf,
+ )
+
+ def __call__(
+ self,
+ text: str,
+ image: Union[str, Image.Image],
+ time_length: int = 5,
+ frame_num: Optional[int] = None,
+ seed: int = None,
+ num_steps: int = None,
+ guidance_weight: float = None,
+ scheduler_scale: float = 10.0,
+ negative_caption: str = (
+ "Static, 2D cartoon, cartoon, 2d animation, paintings, images, worst quality, "
+ "low quality, ugly, deformed, walking backwards"
+ ),
+ expand_prompts: bool = True,
+ save_path: str = None,
+ progress: bool = True,
+ joint_pass: bool = False,
+ ):
+ return super().__call__(
+ text=text,
+ image=image,
+ time_length=time_length,
+ width=None,
+ height=None,
+ frame_num=frame_num,
+ seed=seed,
+ num_steps=num_steps,
+ guidance_weight=guidance_weight,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ expand_prompts=expand_prompts,
+ save_path=save_path,
+ progress=progress,
+ joint_pass=joint_pass,
+ )
diff --git a/models/kandinsky5/kandinsky/magcache_utils.py b/models/kandinsky5/kandinsky/magcache_utils.py
new file mode 100644
index 0000000..04d95d6
--- /dev/null
+++ b/models/kandinsky5/kandinsky/magcache_utils.py
@@ -0,0 +1,405 @@
+# This is an adaptation of Magcache from https://github.com/Zehong-Ma/MagCache/
+import numpy as np
+import torch
+from types import MethodType
+
+
+def nearest_interp(src_array, target_length):
+ src_length = len(src_array)
+ if target_length == 1:
+ return np.array([src_array[-1]])
+
+ scale = (src_length - 1) / (target_length - 1)
+ mapped_indices = np.round(np.arange(target_length) * scale).astype(int)
+ return src_array[mapped_indices]
+
+
+def _prepare_mag_ratios(mag_ratios, num_steps):
+ if mag_ratios is None:
+ return None
+ mag_ratios = np.array([1.0] * 2 + list(mag_ratios))
+ if len(mag_ratios) != num_steps * 2:
+ mag_ratio_con = nearest_interp(mag_ratios[0::2], num_steps)
+ mag_ratio_ucon = nearest_interp(mag_ratios[1::2], num_steps)
+ mag_ratios = np.concatenate(
+ [mag_ratio_con.reshape(-1, 1), mag_ratio_ucon.reshape(-1, 1)], axis=1
+ ).reshape(-1)
+ return mag_ratios
+
+
+def compute_magcache_threshold(
+ mag_ratios,
+ num_steps,
+ speed_factor,
+ start_step=0,
+ no_cfg=False,
+ magcache_K=2,
+ retention_ratio=0.2,
+):
+ if mag_ratios is None or num_steps <= 0 or speed_factor is None or speed_factor <= 0:
+ return None
+
+ mag_ratios = _prepare_mag_ratios(mag_ratios, num_steps)
+ if mag_ratios is None:
+ return None
+
+ total_calls = num_steps if no_cfg else num_steps * 2
+ target_calls = max(1, int(total_calls / max(speed_factor, 1e-6)))
+ retention_cnt = int(num_steps * 2 * retention_ratio)
+
+ best_threshold = 0.01
+ best_diff = float("inf")
+ best_signed_diff = 0
+
+ threshold = 0.01
+ while threshold <= 0.6:
+ nb_calls = 0
+ accumulated_err = [0.0, 0.0]
+ accumulated_steps = [0, 0]
+ accumulated_ratio = [1.0, 1.0]
+
+ for i in range(total_calls):
+ cnt = i * 2 if no_cfg else i
+ step_no = cnt // 2
+ skip_forward = False
+
+ if cnt >= retention_cnt and (start_step is None or step_no > start_step):
+ stream = cnt % 2
+ cur_mag_ratio = mag_ratios[cnt]
+ accumulated_ratio[stream] *= cur_mag_ratio
+ accumulated_steps[stream] += 1
+ cur_skip_err = np.abs(1 - accumulated_ratio[stream])
+ accumulated_err[stream] += cur_skip_err
+
+ if accumulated_err[stream] < threshold and accumulated_steps[stream] <= magcache_K:
+ skip_forward = True
+ else:
+ accumulated_err[stream] = 0.0
+ accumulated_steps[stream] = 0
+ accumulated_ratio[stream] = 1.0
+
+ if not skip_forward:
+ nb_calls += 1
+
+ signed_diff = target_calls - nb_calls
+ diff = abs(signed_diff)
+ if diff < best_diff:
+ best_threshold = threshold
+ best_diff = diff
+ best_signed_diff = signed_diff
+ elif diff > best_diff:
+ break
+ threshold += 0.01
+
+ nb_calls = target_calls - best_signed_diff
+ achieved_speed = total_calls / max(1, nb_calls)
+ print(
+ f"Mag Cache, best threshold found:{best_threshold:0.2f} "
+ f"with gain x{achieved_speed:0.2f} for a target of x{speed_factor}"
+ )
+ return best_threshold
+
+
+def set_magcache_params(
+ dit,
+ mag_ratios,
+ num_steps,
+ no_cfg,
+ start_step=None,
+ magcache_thresh=None,
+ magcache_K=None,
+ retention_ratio=None,
+):
+ print('using Magcache')
+ dit.forward = MethodType(magcache_forward, dit)
+ dit.cnt = 0
+ dit.num_steps = num_steps * 2
+ dit.magcache_thresh = 0.12 if magcache_thresh is None else magcache_thresh
+ dit.K = 2 if magcache_K is None else magcache_K
+ dit.accumulated_err = [0.0, 0.0]
+ dit.accumulated_steps = [0, 0]
+ dit.accumulated_ratio = [1.0, 1.0]
+ dit.consecutive_skips = [0, 0]
+ dit.magcache_start_step = 0 if start_step is None else int(start_step)
+ dit.retention_ratio = 0.2 if retention_ratio is None else retention_ratio
+ dit.magcache_retention_cnt = int(dit.num_steps * dit.retention_ratio)
+ dit.residual_cache = [None, None]
+ dit.mag_ratios = _prepare_mag_ratios(mag_ratios, num_steps)
+ dit.no_cfg = no_cfg
+
+
+def _magcache_should_skip(dit, cnt):
+ stream = cnt % 2
+ skip_forward = False
+ residual_visual_embed = None
+
+ step_no = cnt // 2
+ if cnt >= dit.magcache_retention_cnt and step_no > dit.magcache_start_step:
+ cur_mag_ratio = dit.mag_ratios[cnt]
+ dit.accumulated_ratio[stream] *= cur_mag_ratio
+ dit.accumulated_steps[stream] += 1
+ cur_skip_err = np.abs(1 - dit.accumulated_ratio[stream])
+ dit.accumulated_err[stream] += cur_skip_err
+
+ if dit.accumulated_err[stream] < dit.magcache_thresh and dit.accumulated_steps[stream] <= dit.K:
+ if getattr(dit, "consecutive_skips", [0, 0])[stream] < dit.K:
+ skip_forward = True
+ residual_visual_embed = dit.residual_cache[stream]
+ else:
+ dit.accumulated_err[stream] = 0.0
+ dit.accumulated_steps[stream] = 0
+ dit.accumulated_ratio[stream] = 1.0
+ else:
+ dit.accumulated_err[stream] = 0.0
+ dit.accumulated_steps[stream] = 0
+ dit.accumulated_ratio[stream] = 1.0
+
+ return skip_forward, residual_visual_embed, stream
+
+
+def _magcache_forward_single(
+ self,
+ x,
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor=(1.0, 1.0, 1.0),
+ sparse_params=None,
+ attention_mask=None
+):
+ text_embed, time_embed, text_rope, visual_embed = self.before_text_transformer_blocks(
+ text_embed, time, pooled_text_embed, x, text_rope_pos)
+ x = None
+ pooled_text_embed = None
+
+ for text_transformer_block in self.text_transformer_blocks:
+ text_embed = text_transformer_block(text_embed, time_embed, text_rope, attention_mask)
+ if self._check_interrupt():
+ return None
+ text_rope = None
+
+ visual_embed, visual_shape, to_fractal, visual_rope = self.before_visual_transformer_blocks(
+ visual_embed, visual_rope_pos, scale_factor, sparse_params)
+ visual_rope_pos = None
+
+ skip_forward, residual_visual_embed, stream = _magcache_should_skip(self, self.cnt)
+
+ if skip_forward and residual_visual_embed is None:
+ skip_forward = False
+
+ if skip_forward:
+ cache = getattr(self, "cache", None)
+ if cache is not None and hasattr(cache, "skipped_steps") and stream == 0:
+ cache.skipped_steps += 1
+ visual_embed = visual_embed + residual_visual_embed
+ if hasattr(self, "consecutive_skips"):
+ self.consecutive_skips[stream] += 1
+ else:
+ if hasattr(self, "consecutive_skips"):
+ self.consecutive_skips[stream] = 0
+ ori_visual_embed = visual_embed.clone()
+ for visual_transformer_block in self.visual_transformer_blocks:
+ visual_embed = visual_transformer_block(visual_embed, text_embed, time_embed,
+ visual_rope, sparse_params, attention_mask)
+ if self._check_interrupt():
+ return None
+ torch.sub(visual_embed, ori_visual_embed, out=ori_visual_embed)
+ residual_visual_embed = ori_visual_embed
+
+ self.residual_cache[stream] = residual_visual_embed
+ visual_rope = None
+
+ x = self.after_blocks(visual_embed, visual_shape, to_fractal, text_embed, time_embed)
+
+ if self.no_cfg:
+ self.cnt += 2
+ else:
+ self.cnt += 1
+
+ if self.cnt >= self.num_steps:
+ self.cnt = 0
+ self.accumulated_ratio = [1.0, 1.0]
+ self.accumulated_err = [0.0, 0.0]
+ self.accumulated_steps = [0, 0]
+ if hasattr(self, "consecutive_skips"):
+ self.consecutive_skips = [0, 0]
+ return x
+
+
+def _magcache_forward_joint(
+ self,
+ x_list,
+ text_embed_list,
+ pooled_text_embed_list,
+ time_list,
+ visual_rope_pos_list,
+ text_rope_pos_list,
+ scale_factor_list,
+ sparse_params_list,
+ attention_mask_list,
+):
+ count = len(x_list)
+ text_embed_list = self._normalize_list(text_embed_list, count)
+ pooled_text_embed_list = self._normalize_list(pooled_text_embed_list, count)
+ time_list = self._normalize_list(time_list, count)
+ visual_rope_pos_list = self._normalize_list(visual_rope_pos_list, count)
+ text_rope_pos_list = self._normalize_list(text_rope_pos_list, count)
+ scale_factor_list = self._normalize_list(scale_factor_list, count)
+ sparse_params_list = self._normalize_list(sparse_params_list, count)
+ attention_mask_list = self._normalize_list(attention_mask_list, count)
+
+ text_embed_out = [None] * count
+ time_embed_out = [None] * count
+ text_rope_out = [None] * count
+ visual_embed_out = [None] * count
+
+ for idx in range(count):
+ text_embed_out[idx], time_embed_out[idx], text_rope_out[idx], visual_embed_out[idx] = (
+ self.before_text_transformer_blocks(
+ text_embed_list[idx],
+ time_list[idx],
+ pooled_text_embed_list[idx],
+ x_list[idx],
+ text_rope_pos_list[idx],
+ )
+ )
+ x_list[idx] = None
+ pooled_text_embed_list[idx] = None
+
+ for text_transformer_block in self.text_transformer_blocks:
+ for idx in range(count):
+ text_embed_out[idx] = text_transformer_block(
+ text_embed_out[idx], time_embed_out[idx], text_rope_out[idx], attention_mask_list[idx]
+ )
+ if self._check_interrupt():
+ return None
+ for idx in range(count):
+ text_rope_out[idx] = None
+
+ visual_shape_list = [None] * count
+ to_fractal_list = [None] * count
+ visual_rope_out = [None] * count
+
+ for idx in range(count):
+ visual_embed_out[idx], visual_shape_list[idx], to_fractal_list[idx], visual_rope_out[idx] = (
+ self.before_visual_transformer_blocks(
+ visual_embed_out[idx],
+ visual_rope_pos_list[idx],
+ scale_factor_list[idx],
+ sparse_params_list[idx],
+ )
+ )
+ visual_rope_pos_list[idx] = None
+
+ stream_ids = [0] * count
+ skip_forward = [False] * count
+ residual_visual_embed = [None] * count
+ ori_visual_embed = [None] * count
+
+ cnt = self.cnt
+ for idx in range(count):
+ stream_ids[idx] = cnt % 2
+ skip_forward[idx], residual_visual_embed[idx], _ = _magcache_should_skip(self, cnt)
+ if skip_forward[idx] and residual_visual_embed[idx] is None:
+ skip_forward[idx] = False
+ if not skip_forward[idx] and hasattr(self, "consecutive_skips"):
+ self.consecutive_skips[stream_ids[idx]] = 0
+ if skip_forward[idx]:
+ if idx == 0:
+ cache = getattr(self, "cache", None)
+ if cache is not None and hasattr(cache, "skipped_steps"):
+ cache.skipped_steps += 1
+ visual_embed_out[idx] = visual_embed_out[idx] + residual_visual_embed[idx]
+ if hasattr(self, "consecutive_skips"):
+ self.consecutive_skips[stream_ids[idx]] += 1
+ else:
+ ori_visual_embed[idx] = visual_embed_out[idx].clone()
+ cnt += 2 if self.no_cfg else 1
+
+ for visual_transformer_block in self.visual_transformer_blocks:
+ for idx in range(count):
+ if skip_forward[idx]:
+ continue
+ visual_embed_out[idx] = visual_transformer_block(
+ visual_embed_out[idx],
+ text_embed_out[idx],
+ time_embed_out[idx],
+ visual_rope_out[idx],
+ sparse_params_list[idx],
+ attention_mask_list[idx],
+ )
+ if self._check_interrupt():
+ return None
+ for idx in range(count):
+ visual_rope_out[idx] = None
+
+ for idx in range(count):
+ if skip_forward[idx]:
+ residual = residual_visual_embed[idx]
+ else:
+ torch.sub(visual_embed_out[idx], ori_visual_embed[idx], out=ori_visual_embed[idx])
+ residual = ori_visual_embed[idx]
+ self.residual_cache[stream_ids[idx]] = residual
+
+ outputs = []
+ for idx in range(count):
+ outputs.append(
+ self.after_blocks(
+ visual_embed_out[idx],
+ visual_shape_list[idx],
+ to_fractal_list[idx],
+ text_embed_out[idx],
+ time_embed_out[idx],
+ )
+ )
+
+ self.cnt = cnt
+ if self.cnt >= self.num_steps:
+ self.cnt = 0
+ self.accumulated_ratio = [1.0, 1.0]
+ self.accumulated_err = [0.0, 0.0]
+ self.accumulated_steps = [0, 0]
+ if hasattr(self, "consecutive_skips"):
+ self.consecutive_skips = [0, 0]
+ return outputs
+
+
+def magcache_forward(
+ self,
+ x,
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor=(1.0, 1.0, 1.0),
+ sparse_params=None,
+ attention_mask=None,
+):
+ if isinstance(x, (list, tuple)):
+ return _magcache_forward_joint(
+ self,
+ list(x),
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor,
+ sparse_params,
+ attention_mask,
+ )
+ return _magcache_forward_single(
+ self,
+ x,
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor=scale_factor,
+ sparse_params=sparse_params,
+ attention_mask=attention_mask,
+ )
diff --git a/models/kandinsky5/kandinsky/models/__init__.py b/models/kandinsky5/kandinsky/models/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/models/kandinsky5/kandinsky/models/attention.py b/models/kandinsky5/kandinsky/models/attention.py
new file mode 100644
index 0000000..1549fff
--- /dev/null
+++ b/models/kandinsky5/kandinsky/models/attention.py
@@ -0,0 +1,68 @@
+import torch
+from shared.attention import pay_attention
+
+
+def _normalize_attention_engine(engine):
+ if engine is None or engine == "auto":
+ return None, None
+ if engine == "flash_attention_2":
+ return "flash", None
+ if engine == "flash_attention_3":
+ return "flash", 3
+ return engine, None
+
+
+def _expand_attention_mask(attention_mask, q_len, num_heads):
+ if attention_mask is None:
+ return None
+ mask = attention_mask
+ if mask.dtype != torch.bool:
+ mask = mask.to(torch.bool)
+ if mask.dim() == 2:
+ mask = mask[:, None, None, :]
+ return mask.expand(-1, q_len, num_heads, -1)
+ if mask.dim() == 3:
+ if mask.shape[1] == q_len:
+ mask = mask[:, :, None, :]
+ return mask.expand(-1, q_len, num_heads, -1)
+ if mask.shape[1] == num_heads:
+ mask = mask[:, None, :, :]
+ return mask.expand(-1, q_len, num_heads, -1)
+ if mask.dim() == 4:
+ if mask.shape[1] == num_heads and mask.shape[2] == q_len:
+ return mask.transpose(1, 2)
+ if mask.shape[2] == num_heads and mask.shape[1] == q_len:
+ return mask
+ return mask
+
+
+def apply_attention(q, k, v, attention_mask=None, force_attention=None, flash_version=None):
+ q_len = q.shape[1]
+ num_heads = q.shape[2]
+ mask = _expand_attention_mask(attention_mask, q_len, num_heads)
+ qkv_list = [q, k, v]
+ return pay_attention(
+ qkv_list,
+ attention_mask=mask,
+ force_attention=force_attention,
+ version=flash_version,
+ )
+
+
+class SelfAttentionEngine:
+ def __init__(self, engine="auto"):
+ self.force_attention, self.flash_version = _normalize_attention_engine(engine)
+
+ def __call__(self, q, k, v, attn_mask=None):
+ return apply_attention(
+ q,
+ k,
+ v,
+ attention_mask=attn_mask,
+ force_attention=self.force_attention,
+ flash_version=self.flash_version,
+ )
+
+ def get_attention(self):
+ return self.__call__
+
diff --git a/models/kandinsky5/kandinsky/models/dit.py b/models/kandinsky5/kandinsky/models/dit.py
new file mode 100644
index 0000000..83f57f8
--- /dev/null
+++ b/models/kandinsky5/kandinsky/models/dit.py
@@ -0,0 +1,380 @@
+import torch
+from torch import nn
+
+from .nn import (
+ TimeEmbeddings,
+ TextEmbeddings,
+ VisualEmbeddings,
+ RoPE1D,
+ RoPE3D,
+ Modulation,
+ MultiheadSelfAttentionEnc,
+ MultiheadSelfAttentionDec,
+ MultiheadCrossAttention,
+ FeedForward,
+ OutLayer,
+)
+from .utils import fractal_flatten, fractal_unflatten
+
+
+class TransformerEncoderBlock(nn.Module):
+ def __init__(self, model_dim, time_dim, ff_dim, head_dim, attention_engine="auto", text_token_padding=False):
+ super().__init__()
+ self.text_modulation = Modulation(time_dim, model_dim, 6)
+
+ self.self_attention_norm = nn.LayerNorm(model_dim, elementwise_affine=False)
+ self.self_attention = MultiheadSelfAttentionEnc(model_dim, head_dim, attention_engine, text_token_padding)
+
+ self.feed_forward_norm = nn.LayerNorm(model_dim, elementwise_affine=False)
+ self.feed_forward = FeedForward(model_dim, ff_dim)
+
+ def _modulate_inplace(self, x, shift, scale, gate):
+ shift = shift.to(dtype=x.dtype)
+ scale = scale.to(dtype=x.dtype)
+ gate = gate.to(dtype=x.dtype)
+ scale.add_(1.0)
+ x.mul_(scale.unsqueeze(1))
+ x.add_(shift.unsqueeze(1))
+ return gate.unsqueeze(1)
+
+ def forward(self, x, time_embed, rope, attention_mask=None):
+ self_attn_params, ff_params = torch.chunk(self.text_modulation(time_embed), 2, dim=-1)
+ shift, scale, gate = torch.chunk(self_attn_params, 3, dim=-1)
+ out = self.self_attention_norm(x)
+ gate = self._modulate_inplace(out, shift, scale, gate)
+ out = self.self_attention(out, rope, attention_mask)
+ x.addcmul_(out, gate)
+ out = None
+
+ shift, scale, gate = torch.chunk(ff_params, 3, dim=-1)
+ out = self.feed_forward_norm(x)
+ gate = self._modulate_inplace(out, shift, scale, gate)
+ out = self.feed_forward(out)
+ x.addcmul_(out, gate)
+ out = None
+ return x
+
+
+class TransformerDecoderBlock(nn.Module):
+ def __init__(self, model_dim, time_dim, ff_dim, head_dim, attention_engine="auto", text_token_padding=False):
+ super().__init__()
+ self.visual_modulation = Modulation(time_dim, model_dim, 9)
+ self.ffn_mult = max(int(round(ff_dim / model_dim)), 1)
+
+ self.self_attention_norm = nn.LayerNorm(model_dim, elementwise_affine=False)
+ self.self_attention = MultiheadSelfAttentionDec(model_dim, head_dim, attention_engine)
+
+ self.cross_attention_norm = nn.LayerNorm(model_dim, elementwise_affine=False)
+ self.cross_attention = MultiheadCrossAttention(model_dim, head_dim, attention_engine,text_token_padding)
+
+ self.feed_forward_norm = nn.LayerNorm(model_dim, elementwise_affine=False)
+ self.feed_forward = FeedForward(model_dim, ff_dim)
+
+ def _modulate_inplace(self, x, shift, scale, gate):
+ shift = shift.to(dtype=x.dtype)
+ scale = scale.to(dtype=x.dtype)
+ gate = gate.to(dtype=x.dtype)
+ scale.add_(1.0)
+ x.mul_(scale.unsqueeze(1))
+ x.add_(shift.unsqueeze(1))
+ return gate.unsqueeze(1)
+
+ def _apply_ffn_chunked(self, ffn_in: torch.Tensor) -> torch.Tensor:
+ if self.ffn_mult <= 1:
+ return self.feed_forward(ffn_in)
+ seq_len = ffn_in.shape[1]
+ ffn_flat = ffn_in.reshape(-1, ffn_in.shape[-1])
+ chunk_size = max(seq_len // self.ffn_mult, 1)
+ for chunk in torch.split(ffn_flat, chunk_size, dim=0):
+ chunk[...] = self.feed_forward(chunk)
+ return ffn_in
+
+ def forward(self, visual_embed, text_embed, time_embed, rope, sparse_params, attention_mask=None):
+ self_attn_params, cross_attn_params, ff_params = torch.chunk(
+ self.visual_modulation(time_embed), 3, dim=-1
+ )
+ shift, scale, gate = torch.chunk(self_attn_params, 3, dim=-1)
+ visual_out = self.self_attention_norm(visual_embed)
+ gate = self._modulate_inplace(visual_out, shift, scale, gate)
+ visual_out = self.self_attention(visual_out, rope, sparse_params)
+ visual_embed.addcmul_(visual_out, gate)
+ visual_out = None
+
+ shift, scale, gate = torch.chunk(cross_attn_params, 3, dim=-1)
+ visual_out = self.cross_attention_norm(visual_embed)
+ gate = self._modulate_inplace(visual_out, shift, scale, gate)
+ visual_out = self.cross_attention(visual_out, text_embed, attention_mask)
+ visual_embed.addcmul_(visual_out, gate)
+ visual_out = None
+
+ shift, scale, gate = torch.chunk(ff_params, 3, dim=-1)
+ visual_out = self.feed_forward_norm(visual_embed)
+ gate = self._modulate_inplace(visual_out, shift, scale, gate)
+ visual_out = self._apply_ffn_chunked(visual_out)
+ visual_embed.addcmul_(visual_out, gate)
+ visual_out = None
+ return visual_embed
+
+
+class DiffusionTransformer3D(nn.Module):
+ def __init__(
+ self,
+ in_visual_dim=4,
+ in_text_dim=3584,
+ in_text_dim2=768,
+ time_dim=512,
+ out_visual_dim=4,
+ patch_size=(1, 2, 2),
+ model_dim=2048,
+ ff_dim=5120,
+ num_text_blocks=2,
+ num_visual_blocks=32,
+ axes_dims=(16, 24, 24),
+ visual_cond=False,
+ attention_engine="auto",
+ instruct_type=None,
+ text_token_padding=False
+ ):
+ super().__init__()
+ self.instruct_type = instruct_type
+ head_dim = sum(axes_dims)
+ self.in_visual_dim = in_visual_dim
+ self.model_dim = model_dim
+ self.patch_size = patch_size
+ self.visual_cond = visual_cond
+ self.text_token_padding = text_token_padding
+
+ visual_embed_dim = 2 * in_visual_dim + 1 if visual_cond or instruct_type=='channel' else in_visual_dim
+ self.time_embeddings = TimeEmbeddings(model_dim, time_dim)
+ self.text_embeddings = TextEmbeddings(in_text_dim, model_dim)
+ self.pooled_text_embeddings = TextEmbeddings(in_text_dim2, time_dim)
+ self.visual_embeddings = VisualEmbeddings(visual_embed_dim, model_dim, patch_size)
+
+ self.text_rope_embeddings = RoPE1D(head_dim)
+ self.text_transformer_blocks = nn.ModuleList(
+ [
+ TransformerEncoderBlock(model_dim, time_dim, ff_dim, head_dim, attention_engine, text_token_padding)
+ for _ in range(num_text_blocks)
+ ]
+ )
+
+ self.visual_rope_embeddings = RoPE3D(axes_dims)
+ self.visual_transformer_blocks = nn.ModuleList(
+ [
+ TransformerDecoderBlock(model_dim, time_dim, ff_dim, head_dim, attention_engine, text_token_padding)
+ for _ in range(num_visual_blocks)
+ ]
+ )
+
+ self.out_layer = OutLayer(model_dim, time_dim, out_visual_dim, patch_size)
+ self._interrupt_check = None
+
+ def before_text_transformer_blocks(self, text_embed, time, pooled_text_embed, x,
+ text_rope_pos):
+ text_embed = self.text_embeddings(text_embed)
+ time_embed = self.time_embeddings(time)
+ time_embed = time_embed + self.pooled_text_embeddings(pooled_text_embed)
+ visual_embed = self.visual_embeddings(x)
+ text_rope = self.text_rope_embeddings(text_rope_pos)
+ return text_embed, time_embed, text_rope, visual_embed
+
+ def before_visual_transformer_blocks(self, visual_embed, visual_rope_pos, scale_factor,
+ sparse_params):
+ visual_shape = visual_embed.shape[:-1]
+ visual_rope = self.visual_rope_embeddings(visual_shape, visual_rope_pos, scale_factor)
+ to_fractal = sparse_params["to_fractal"] if sparse_params is not None else False
+ visual_embed, visual_rope = fractal_flatten(visual_embed, visual_rope, visual_shape,
+ block_mask=to_fractal)
+ return visual_embed.unsqueeze(0), visual_shape, to_fractal, visual_rope
+
+ def after_blocks(self, visual_embed, visual_shape, to_fractal, text_embed, time_embed):
+ visual_embed = fractal_unflatten(visual_embed, visual_shape, block_mask=to_fractal)
+ x = self.out_layer(visual_embed, text_embed, time_embed)
+ return x
+
+ def _check_interrupt(self):
+ if self._interrupt_check is None:
+ return False
+ return bool(self._interrupt_check())
+
+ @staticmethod
+ def _normalize_list(value, count):
+ if isinstance(value, (list, tuple)):
+ if len(value) != count:
+ raise ValueError(f"Expected {count} items, got {len(value)}.")
+ return list(value)
+ return [value] * count
+
+ def _forward_single(
+ self,
+ x,
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor,
+ sparse_params,
+ attention_mask,
+ ):
+ text_embed, time_embed, text_rope, visual_embed = self.before_text_transformer_blocks(
+ text_embed, time, pooled_text_embed, x, text_rope_pos
+ )
+ x = None
+ pooled_text_embed = None
+
+ for text_transformer_block in self.text_transformer_blocks:
+ text_embed = text_transformer_block(text_embed, time_embed, text_rope, attention_mask)
+ if self._check_interrupt():
+ return None
+ text_rope = None
+
+ visual_embed, visual_shape, to_fractal, visual_rope = self.before_visual_transformer_blocks(
+ visual_embed, visual_rope_pos, scale_factor, sparse_params
+ )
+ visual_rope_pos = None
+
+ for visual_transformer_block in self.visual_transformer_blocks:
+ visual_embed = visual_transformer_block(
+ visual_embed, text_embed, time_embed, visual_rope, sparse_params, attention_mask
+ )
+ if self._check_interrupt():
+ return None
+ visual_rope = None
+
+ return self.after_blocks(visual_embed, visual_shape, to_fractal, text_embed, time_embed)
+
+ def _forward_joint(
+ self,
+ x_list,
+ text_embed_list,
+ pooled_text_embed_list,
+ time_list,
+ visual_rope_pos_list,
+ text_rope_pos_list,
+ scale_factor_list,
+ sparse_params_list,
+ attention_mask_list,
+ ):
+ count = len(x_list)
+ text_embed_list = self._normalize_list(text_embed_list, count)
+ pooled_text_embed_list = self._normalize_list(pooled_text_embed_list, count)
+ time_list = self._normalize_list(time_list, count)
+ visual_rope_pos_list = self._normalize_list(visual_rope_pos_list, count)
+ text_rope_pos_list = self._normalize_list(text_rope_pos_list, count)
+ scale_factor_list = self._normalize_list(scale_factor_list, count)
+ sparse_params_list = self._normalize_list(sparse_params_list, count)
+ attention_mask_list = self._normalize_list(attention_mask_list, count)
+
+ text_embed_out = [None] * count
+ time_embed_out = [None] * count
+ text_rope_out = [None] * count
+ visual_embed_out = [None] * count
+
+ for idx in range(count):
+ text_embed_out[idx], time_embed_out[idx], text_rope_out[idx], visual_embed_out[idx] = (
+ self.before_text_transformer_blocks(
+ text_embed_list[idx],
+ time_list[idx],
+ pooled_text_embed_list[idx],
+ x_list[idx],
+ text_rope_pos_list[idx],
+ )
+ )
+ x_list[idx] = None
+ pooled_text_embed_list[idx] = None
+
+ for text_transformer_block in self.text_transformer_blocks:
+ for idx in range(count):
+ text_embed_out[idx] = text_transformer_block(
+ text_embed_out[idx], time_embed_out[idx], text_rope_out[idx], attention_mask_list[idx]
+ )
+ if self._check_interrupt():
+ return None
+ for idx in range(count):
+ text_rope_out[idx] = None
+
+ visual_shape_list = [None] * count
+ to_fractal_list = [None] * count
+ visual_rope_out = [None] * count
+
+ for idx in range(count):
+ visual_embed_out[idx], visual_shape_list[idx], to_fractal_list[idx], visual_rope_out[idx] = (
+ self.before_visual_transformer_blocks(
+ visual_embed_out[idx],
+ visual_rope_pos_list[idx],
+ scale_factor_list[idx],
+ sparse_params_list[idx],
+ )
+ )
+ visual_rope_pos_list[idx] = None
+
+ for visual_transformer_block in self.visual_transformer_blocks:
+ for idx in range(count):
+ visual_embed_out[idx] = visual_transformer_block(
+ visual_embed_out[idx],
+ text_embed_out[idx],
+ time_embed_out[idx],
+ visual_rope_out[idx],
+ sparse_params_list[idx],
+ attention_mask_list[idx],
+ )
+ if self._check_interrupt():
+ return None
+ for idx in range(count):
+ visual_rope_out[idx] = None
+
+ outputs = []
+ for idx in range(count):
+ outputs.append(
+ self.after_blocks(
+ visual_embed_out[idx],
+ visual_shape_list[idx],
+ to_fractal_list[idx],
+ text_embed_out[idx],
+ time_embed_out[idx],
+ )
+ )
+ return outputs
+
+ def forward(
+ self,
+ x,
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor=(1.0, 1.0, 1.0),
+ sparse_params=None,
+ attention_mask=None,
+ ):
+ if isinstance(x, (list, tuple)):
+ return self._forward_joint(
+ list(x),
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor,
+ sparse_params,
+ attention_mask,
+ )
+
+ return self._forward_single(
+ x,
+ text_embed,
+ pooled_text_embed,
+ time,
+ visual_rope_pos,
+ text_rope_pos,
+ scale_factor,
+ sparse_params,
+ attention_mask,
+ )
+
+
+def get_dit(conf, text_token_padding=False):
+ dit = DiffusionTransformer3D(**conf, text_token_padding=text_token_padding)
+ return dit
diff --git a/models/kandinsky5/kandinsky/models/nn.py b/models/kandinsky5/kandinsky/models/nn.py
new file mode 100644
index 0000000..317b3df
--- /dev/null
+++ b/models/kandinsky5/kandinsky/models/nn.py
@@ -0,0 +1,420 @@
+import math
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+try:
+ from torch.nn.attention.flex_attention import flex_attention
+except Exception:
+ flex_attention = None
+
+from .utils import get_freqs, nablaT_v2
+from .attention import SelfAttentionEngine
+
[email protected](device_type="cuda", dtype=torch.float32)
+def apply_scale_shift_norm(norm, x, scale, shift):
+ return (norm(x) * (scale + 1.0) + shift).to(torch.bfloat16)
+
[email protected](device_type="cuda", dtype=torch.float32)
+def apply_gate_sum(x, out, gate):
+ return (x + gate * out).to(torch.bfloat16)
+
[email protected](device_type="cuda", enabled=False)
+def apply_rotary(x, rope):
+ x_ = x.reshape(*x.shape[:-1], -1, 1, 2).to(torch.float32)
+ x_out = (rope * x_).sum(dim=-1)
+ return x_out.reshape(*x.shape).to(torch.bfloat16)
+
[email protected](device_type="cuda", enabled=False)
+def apply_rotary_inplace(x_list, rope):
+ x_in = x_list[0]
+ dtype = x_in.dtype
+ x = x_in.float().reshape(*x_in.shape[:-1], -1, 2)
+ x_in = None
+ x_list.clear()
+
+ r00 = rope[..., 0, 0]
+ r01 = rope[..., 0, 1]
+ r10 = rope[..., 1, 0]
+ r11 = rope[..., 1, 1]
+
+ x0 = x[..., 0]
+ x1 = x[..., 1]
+ x0_orig = x0.clone()
+ x0.mul_(r00)
+ x0.addcmul_(x1, r01)
+ x1.mul_(r11)
+ x1.addcmul_(x0_orig, r10)
+
+ return x.flatten(3).to(dtype)
+
+
+class TimeEmbeddings(nn.Module):
+ def __init__(self, model_dim, time_dim, max_period=10000.0):
+ super().__init__()
+ assert model_dim % 2 == 0
+ self.model_dim = model_dim
+ self.max_period = max_period
+ self.register_buffer(
+ "freqs", get_freqs(model_dim // 2, max_period), persistent=False
+ )
+ self.in_layer = nn.Linear(model_dim, time_dim, bias=True)
+ self.activation = nn.SiLU()
+ self.out_layer = nn.Linear(time_dim, time_dim, bias=True)
+
+ @torch.autocast(device_type="cuda", dtype=torch.float32)
+ def forward(self, time):
+ args = torch.outer(time, self.freqs.to(device=time.device))
+ time_embed = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+ time_embed = self.out_layer(self.activation(self.in_layer(time_embed)))
+ return time_embed
+
+
+class TextEmbeddings(nn.Module):
+ def __init__(self, text_dim, model_dim):
+ super().__init__()
+ self.in_layer = nn.Linear(text_dim, model_dim, bias=True)
+ self.norm = nn.LayerNorm(model_dim, elementwise_affine=True)
+
+ def forward(self, text_embed):
+ text_embed = self.in_layer(text_embed)
+ return self.norm(text_embed).type_as(text_embed)
+
+
+class VisualEmbeddings(nn.Module):
+ def __init__(self, visual_dim, model_dim, patch_size):
+ super().__init__()
+ self.patch_size = patch_size
+ self.in_layer = nn.Linear(math.prod(patch_size) * visual_dim, model_dim)
+
+ def forward(self, x):
+ duration, height, width, dim = x.shape
+ x = (
+ x.view(
+ duration // self.patch_size[0],
+ self.patch_size[0],
+ height // self.patch_size[1],
+ self.patch_size[1],
+ width // self.patch_size[2],
+ self.patch_size[2],
+ dim,
+ )
+ .permute(0, 2, 4, 1, 3, 5, 6)
+ .flatten(3, 6)
+ )
+ return self.in_layer(x)
+
+
+class RoPE1D(nn.Module):
+ def __init__(self, dim, max_pos=1024, max_period=10000.0):
+ super().__init__()
+ self.max_period = max_period
+ self.dim = dim
+ self.max_pos = max_pos
+ freq = get_freqs(dim // 2, max_period)
+ pos = torch.arange(max_pos, dtype=freq.dtype)
+ self.register_buffer(f"args", torch.outer(pos, freq), persistent=False)
+
+ @torch.autocast(device_type="cuda", enabled=False)
+ def forward(self, pos):
+ args = self.args[pos]
+ cosine = torch.cos(args)
+ sine = torch.sin(args)
+ rope = torch.stack([cosine, -sine, sine, cosine], dim=-1)
+ rope = rope.view(*rope.shape[:-1], 2, 2)
+ return rope.unsqueeze(-4)
+
+
+class RoPE3D(nn.Module):
+ def __init__(self, axes_dims, max_pos=(128, 128, 128), max_period=10000.0):
+ super().__init__()
+ self.axes_dims = axes_dims
+ self.max_pos = max_pos
+ self.max_period = max_period
+
+ for i, (axes_dim, ax_max_pos) in enumerate(zip(axes_dims, max_pos)):
+ freq = get_freqs(axes_dim // 2, max_period)
+ pos = torch.arange(ax_max_pos, dtype=freq.dtype)
+ self.register_buffer(f"args_{i}", torch.outer(pos, freq), persistent=False)
+
+ @torch.autocast(device_type="cuda", enabled=False)
+ def forward(self, shape, pos, scale_factor=(1.0, 1.0, 1.0)):
+ duration, height, width = shape
+ args_t = self.args_0[pos[0]] / scale_factor[0]
+ args_h = self.args_1[pos[1]] / scale_factor[1]
+ args_w = self.args_2[pos[2]] / scale_factor[2]
+
+ args = torch.cat(
+ [
+ args_t.view(duration, 1, 1, -1).repeat(1, height, width, 1),
+ args_h.view(1, height, 1, -1).repeat(duration, 1, width, 1),
+ args_w.view(1, 1, width, -1).repeat(duration, height, 1, 1),
+ ],
+ dim=-1,
+ )
+ cosine = torch.cos(args)
+ sine = torch.sin(args)
+ rope = torch.stack([cosine, -sine, sine, cosine], dim=-1)
+ rope = rope.view(*rope.shape[:-1], 2, 2)
+ return rope.unsqueeze(-4)
+
+
+class Modulation(nn.Module):
+ def __init__(self, time_dim, model_dim, num_params):
+ super().__init__()
+ self.activation = nn.SiLU()
+ self.out_layer = nn.Linear(time_dim, num_params * model_dim)
+ self.out_layer.weight.data.zero_()
+ self.out_layer.bias.data.zero_()
+
+ @torch.autocast(device_type="cuda", dtype=torch.float32)
+ def forward(self, x):
+ return self.out_layer(self.activation(x))
+
+
+class MultiheadSelfAttentionEnc(nn.Module):
+ def __init__(self, num_channels, head_dim, attention_engine="auto", text_token_padding=False):
+ super().__init__()
+ assert num_channels % head_dim == 0
+ self.num_heads = num_channels // head_dim
+
+ self.to_query = nn.Linear(num_channels, num_channels, bias=True)
+ self.to_key = nn.Linear(num_channels, num_channels, bias=True)
+ self.to_value = nn.Linear(num_channels, num_channels, bias=True)
+ self.query_norm = nn.RMSNorm(head_dim)
+ self.key_norm = nn.RMSNorm(head_dim)
+
+ self.out_layer = nn.Linear(num_channels, num_channels, bias=True)
+ if text_token_padding:
+ self.attn_engine = SelfAttentionEngine("sdpa")
+ else:
+ self.attn_engine = SelfAttentionEngine(attention_engine)
+
+ def get_qkv(self, x):
+ query = self.to_query(x)
+ key = self.to_key(x)
+ value = self.to_value(x)
+
+ shape = query.shape[:-1]
+ query = query.reshape(*shape, self.num_heads, -1)
+ key = key.reshape(*shape, self.num_heads, -1)
+ value = value.reshape(*shape, self.num_heads, -1)
+
+ return query, key, value
+
+ def norm_qk(self, q, k):
+ q = self.query_norm(q.float()).type_as(q)
+ k = self.key_norm(k.float()).type_as(k)
+ return q, k
+
+ def scaled_dot_product_attention(self, query, key, value, attention_mask=None):
+ out = self.attn_engine.get_attention()(query, key, value, attention_mask)
+ return out.flatten(-2, -1)
+
+ def out_l(self, x):
+ return self.out_layer(x)
+
+ def forward(self, x, rope, attention_mask=None):
+ query, key, value = self.get_qkv(x)
+ query, key = self.norm_qk(query, key)
+ q_list = [query]
+ query = None
+ query = apply_rotary_inplace(q_list, rope)
+ k_list = [key]
+ key = None
+ key = apply_rotary_inplace(k_list, rope)
+
+ out = self.scaled_dot_product_attention(query, key, value, attention_mask)
+ query = key = value = None
+ out = self.out_l(out)
+ return out
+
+class MultiheadSelfAttentionDec(nn.Module):
+ def __init__(self, num_channels, head_dim, attention_engine="auto"):
+ super().__init__()
+ assert num_channels % head_dim == 0
+ self.num_heads = num_channels // head_dim
+
+ self.to_query = nn.Linear(num_channels, num_channels, bias=True)
+ self.to_key = nn.Linear(num_channels, num_channels, bias=True)
+ self.to_value = nn.Linear(num_channels, num_channels, bias=True)
+ self.query_norm = nn.RMSNorm(head_dim)
+ self.key_norm = nn.RMSNorm(head_dim)
+
+ self.out_layer = nn.Linear(num_channels, num_channels, bias=True)
+
+ self.attn_engine = SelfAttentionEngine(attention_engine)
+
+ def get_qkv(self, x):
+ query = self.to_query(x)
+ key = self.to_key(x)
+ value = self.to_value(x)
+
+ shape = query.shape[:-1]
+ query = query.reshape(*shape, self.num_heads, -1)
+ key = key.reshape(*shape, self.num_heads, -1)
+ value = value.reshape(*shape, self.num_heads, -1)
+
+ return query, key, value
+
+ def norm_qk(self, q, k):
+ q = self.query_norm(q.float()).type_as(q)
+ k = self.key_norm(k.float()).type_as(k)
+ return q, k
+
+ def attention(self, query, key, value):
+ out = self.attn_engine.get_attention()(query, key, value)
+ return out.flatten(-2, -1)
+
+ def nabla(self, query, key, value, sparse_params=None):
+ if flex_attention is None:
+ raise RuntimeError("flex_attention is not available.")
+ query = query.transpose(1, 2).contiguous()
+ key = key.transpose(1, 2).contiguous()
+ value = value.transpose(1, 2).contiguous()
+ block_mask = nablaT_v2(
+ query,
+ key,
+ sparse_params["sta_mask"],
+ thr=sparse_params["P"],
+ )
+ out = (
+ flex_attention(
+ query,
+ key,
+ value,
+ block_mask=block_mask
+ )
+ .transpose(1, 2)
+ .contiguous()
+ )
+ out = out.flatten(-2, -1)
+ return out
+
+ def out_l(self, x):
+ return self.out_layer(x)
+
+ def forward(self, x, rope, sparse_params=None):
+ query, key, value = self.get_qkv(x)
+ query, key = self.norm_qk(query, key)
+ q_list = [query]
+ query = None
+ query = apply_rotary_inplace(q_list, rope)
+ k_list = [key]
+ key = None
+ key = apply_rotary_inplace(k_list, rope)
+
+ if sparse_params is not None and flex_attention is not None:
+ try:
+ out = self.nabla(query, key, value, sparse_params=sparse_params)
+ except RuntimeError:
+ out = self.attention(query, key, value)
+ else:
+ out = self.attention(query, key, value)
+
+ query = key = value = None
+ out = self.out_l(out)
+ return out
+
+
+class MultiheadCrossAttention(nn.Module):
+ def __init__(self, num_channels, head_dim, attention_engine="auto", text_token_padding=False):
+ super().__init__()
+ assert num_channels % head_dim == 0
+ self.num_heads = num_channels // head_dim
+
+ self.to_query = nn.Linear(num_channels, num_channels, bias=True)
+ self.to_key = nn.Linear(num_channels, num_channels, bias=True)
+ self.to_value = nn.Linear(num_channels, num_channels, bias=True)
+ self.query_norm = nn.RMSNorm(head_dim)
+ self.key_norm = nn.RMSNorm(head_dim)
+
+ self.out_layer = nn.Linear(num_channels, num_channels, bias=True)
+
+ if text_token_padding:
+ self.attn_engine = SelfAttentionEngine("sdpa")
+ else:
+ self.attn_engine = SelfAttentionEngine(attention_engine)
+
+ def get_qkv(self, x, cond):
+ query = self.to_query(x)
+ key = self.to_key(cond)
+ value = self.to_value(cond)
+
+ shape, cond_shape = query.shape[:-1], key.shape[:-1]
+ query = query.reshape(*shape, self.num_heads, -1)
+ key = key.reshape(*cond_shape, self.num_heads, -1)
+ value = value.reshape(*cond_shape, self.num_heads, -1)
+
+ return query, key, value
+
+ def norm_qk(self, q, k):
+ q = self.query_norm(q.float()).type_as(q)
+ k = self.key_norm(k.float()).type_as(k)
+ return q, k
+
+ def attention(self, query, key, value, attention_mask=None):
+ out = self.attn_engine.get_attention()(query, key, value, attention_mask)
+ return out.flatten(-2, -1)
+
+ def out_l(self, x):
+ return self.out_layer(x)
+
+ def forward(self, x, cond, attention_mask=None):
+ query, key, value = self.get_qkv(x, cond)
+ query, key = self.norm_qk(query, key)
+
+ out = self.attention(query, key, value, attention_mask)
+ query = key = value = None
+ out = self.out_l(out)
+ return out
+
+
+class FeedForward(nn.Module):
+ def __init__(self, dim, ff_dim):
+ super().__init__()
+ self.in_layer = nn.Linear(dim, ff_dim, bias=False)
+ self.activation = nn.GELU()
+ self.out_layer = nn.Linear(ff_dim, dim, bias=False)
+
+ def forward(self, x):
+ return self.out_layer(self.activation(self.in_layer(x)))
+
+
+class OutLayer(nn.Module):
+ def __init__(self, model_dim, time_dim, visual_dim, patch_size):
+ super().__init__()
+ self.patch_size = patch_size
+ self.modulation = Modulation(time_dim, model_dim, 2)
+ self.norm = nn.LayerNorm(model_dim, elementwise_affine=False)
+ self.out_layer = nn.Linear(
+ model_dim, math.prod(patch_size) * visual_dim, bias=True
+ )
+
+ def forward(self, visual_embed, text_embed, time_embed):
+ shift, scale = torch.chunk(self.modulation(time_embed), 2, dim=-1)
+ visual_embed = apply_scale_shift_norm(
+ self.norm,
+ visual_embed,
+ scale[:, None, None],
+ shift[:, None, None],
+ ).type_as(visual_embed)
+ x = self.out_layer(visual_embed)
+
+ duration, height, width, _ = x.shape
+ x = (
+ x.view(
+ duration,
+ height,
+ width,
+ -1,
+ self.patch_size[0],
+ self.patch_size[1],
+ self.patch_size[2],
+ )
+ .permute(0, 4, 1, 5, 2, 6, 3)
+ .flatten(0, 1)
+ .flatten(1, 2)
+ .flatten(2, 3)
+ )
+ return x
diff --git a/models/kandinsky5/kandinsky/models/parallelize.py b/models/kandinsky5/kandinsky/models/parallelize.py
new file mode 100644
index 0000000..9dc469b
--- /dev/null
+++ b/models/kandinsky5/kandinsky/models/parallelize.py
@@ -0,0 +1,185 @@
+from torch.distributed._tensor import Replicate, Shard
+from torch.distributed.tensor.parallel import (
+ ColwiseParallel,
+ PrepareModuleInput,
+ PrepareModuleOutput,
+ RowwiseParallel,
+ SequenceParallel,
+ parallelize_module,
+)
+
+
+def parallelize_dit(model, tp_mesh):
+ if tp_mesh.size() > 1:
+ plan = {
+ "in_layer": ColwiseParallel(),
+ "out_layer": RowwiseParallel(
+ output_layouts=Replicate(),
+ ),
+ }
+ parallelize_module(model.time_embeddings, tp_mesh, plan)
+
+ plan = {
+ "in_layer": ColwiseParallel(
+ output_layouts=Replicate(),
+ )
+ }
+ parallelize_module(model.text_embeddings, tp_mesh, plan)
+ parallelize_module(model.pooled_text_embeddings, tp_mesh, plan)
+ parallelize_module(model.visual_embeddings, tp_mesh, plan)
+
+ for visual_transformer_block in model.visual_transformer_blocks:
+ plan = {
+ "visual_modulation": PrepareModuleInput(
+ input_layouts=(None),
+ desired_input_layouts=(Replicate()),
+ ),
+ "visual_modulation.out_layer": ColwiseParallel(
+ output_layouts=Replicate(),
+ ),
+ "self_attention_norm": SequenceParallel(
+ sequence_dim=0, use_local_output=True
+ ),
+ "self_attention.to_query": ColwiseParallel(
+ input_layouts=Replicate(),
+ ),
+ "self_attention.to_key": ColwiseParallel(
+ input_layouts=Replicate(),
+ ),
+ "self_attention.to_value": ColwiseParallel(
+ input_layouts=Replicate(),
+ ),
+ "self_attention.query_norm": SequenceParallel(
+ sequence_dim=0, use_local_output=True
+ ),
+ "self_attention.key_norm": SequenceParallel(
+ sequence_dim=0, use_local_output=True
+ ),
+ "self_attention.out_layer": RowwiseParallel(
+ output_layouts=Replicate(),
+ ),
+ "cross_attention_norm": SequenceParallel(
+ sequence_dim=0, use_local_output=True
+ ),
+ "cross_attention.to_query": ColwiseParallel(
+ input_layouts=Replicate(),
+ ),
+ "cross_attention.to_key": ColwiseParallel(
+ input_layouts=Replicate(),
+ ),
+ "cross_attention.to_value": ColwiseParallel(
+ input_layouts=Replicate(),
+ ),
+ "cross_attention.query_norm": SequenceParallel(
+ sequence_dim=0, use_local_output=True
+ ),
+ "cross_attention.key_norm": SequenceParallel(
+ sequence_dim=0, use_local_output=True
+ ),
+ "cross_attention.out_layer": RowwiseParallel(
+ output_layouts=Replicate(),
+ ),
+ "feed_forward_norm": SequenceParallel(
+ sequence_dim=0, use_local_output=True
+ ),
+ "feed_forward.in_layer": ColwiseParallel(),
+ "feed_forward.out_layer": RowwiseParallel(),
+ }
+ self_attn = visual_transformer_block.self_attention
+ self_attn.num_heads = self_attn.num_heads // tp_mesh.size()
+
+ cross_attn = visual_transformer_block.cross_attention
+ cross_attn.num_heads = cross_attn.num_heads // tp_mesh.size()
+
+ parallelize_module(visual_transformer_block, tp_mesh, plan)
+
+ plan = {
+ "out_layer": ColwiseParallel(
+ output_layouts=Replicate(),
+ ),
+ }
+ parallelize_module(model.out_layer, tp_mesh, plan)
+
+ return model
+
+
+def parallelize_seq(model, tp_mesh):
+ if tp_mesh.size() > 1:
+ plan_in = {
+ "out_layer": PrepareModuleInput(
+ input_layouts=(Replicate(), None, None),
+ desired_input_layouts=(Shard(1), None, None),
+ use_local_output=True
+ ),
+ }
+ parallelize_module(model, tp_mesh, plan_in)
+ plan_out = {
+ "visual_embeddings": PrepareModuleOutput(
+ output_layouts=(Shard(1)),
+ desired_output_layouts=(Replicate()),
+ )
+ }
+ parallelize_module(model, tp_mesh, plan_out)
+
+ for i, block in enumerate(model.visual_transformer_blocks):
+ plan = {
+ "self_attention_norm": SequenceParallel(sequence_dim=0, use_local_output=True),
+ "self_attention.to_query": PrepareModuleOutput(
+ output_layouts=(Shard(0)), desired_output_layouts=(Shard(-1))
+ ),
+ "self_attention.to_key": PrepareModuleOutput(
+ output_layouts=(Shard(0)), desired_output_layouts=(Shard(-1))
+ ),
+ "self_attention.to_value": PrepareModuleOutput(
+ output_layouts=(Shard(0)), desired_output_layouts=(Shard(-1))
+ ),
+ "self_attention.out_layer": PrepareModuleInput(
+ input_layouts=(Shard(-1)),
+ desired_input_layouts=(Shard(0)),
+ use_local_output=True,
+ ),
+ "cross_attention_norm": SequenceParallel(sequence_dim=0, use_local_output=True),
+ "cross_attention.to_query": PrepareModuleOutput(
+ output_layouts=(Shard(0)), desired_output_layouts=(Shard(-1))
+ ),
+ "cross_attention.to_key": PrepareModuleOutput(
+ output_layouts=(Replicate()), desired_output_layouts=(Shard(-1))
+ ),
+ "cross_attention.to_value": PrepareModuleOutput(
+ output_layouts=(Replicate()), desired_output_layouts=(Shard(-1))
+ ),
+ "cross_attention.out_layer": PrepareModuleInput(
+ input_layouts=(Shard(-1)),
+ desired_input_layouts=(Shard(0)),
+ use_local_output=True,
+ ),
+ "feed_forward_norm": SequenceParallel(sequence_dim=0, use_local_output=True),
+ }
+ self_attn = block.self_attention
+ self_attn.num_heads = self_attn.num_heads // tp_mesh.size()
+ cross_attn = block.cross_attention
+ cross_attn.num_heads = cross_attn.num_heads // tp_mesh.size()
+ parallelize_module(block, tp_mesh, plan)
+ #shard input of first block and idx for all blocks
+ if i == 0:
+ parallelize_module(
+ block,
+ tp_mesh,
+ PrepareModuleInput(
+ input_layouts=(Replicate(),None,None,None,None, None),
+ desired_input_layouts=(Shard(0),None,None,None,None, None),
+ use_local_output=True,
+ ),
+ )
+
+ if i == len(model.visual_transformer_blocks)-1:
+ parallelize_module(
+ block,
+ tp_mesh,
+ PrepareModuleOutput(
+ output_layouts=(Shard(0)),
+ desired_output_layouts=(Replicate())
+ ),
+ )
+
+ return model
diff --git a/models/kandinsky5/kandinsky/models/text_embedders.py b/models/kandinsky5/kandinsky/models/text_embedders.py
new file mode 100644
index 0000000..c781122
--- /dev/null
+++ b/models/kandinsky5/kandinsky/models/text_embedders.py
@@ -0,0 +1,242 @@
+import os
+import torch
+from transformers import (
+ Qwen2_5_VLForConditionalGeneration,
+ AutoProcessor,
+ CLIPTextModel,
+ CLIPTokenizer,
+ BitsAndBytesConfig
+)
+
+from .utils import freeze
+from mmgp import offload
+import torchvision.transforms.functional as F
+
+
+class ClipTextEmbedder:
+ def __init__(self, conf, device):
+ self.model = CLIPTextModel.from_pretrained(conf.checkpoint_path).to(device)
+ self.model = freeze(self.model)
+ self.tokenizer = CLIPTokenizer.from_pretrained(conf.checkpoint_path)
+ self.max_length = conf.max_length
+
+ def __call__(self, texts):
+ inputs = self.tokenizer(
+ texts,
+ max_length=self.max_length,
+ truncation=True,
+ add_special_tokens=True,
+ padding="max_length",
+ return_tensors="pt",
+ ).to(self.model.device)
+
+ with torch.no_grad():
+ pooled_embed = self.model(**inputs)["pooler_output"]
+ return pooled_embed
+
+
+class Qwen2_5_VLTextEmbedder:
+ INSTRUCTION_I2V_EXPAND = """You are a prompt beautifier that transforms short user video descriptions into rich, detailed English prompts specifically optimized for video generation models.
+ Here are some example descriptions from the dataset that the model was trained:
+ 1. "Create a video showing a nighttime urban driving scene from inside a car. The driver is focused on the road ahead, with the city lights visible through the windshield. The GPS device on the dashboard continues to display navigation information. The camera remains steady, capturing the interior of the car and the changing street view outside as the vehicle moves forward. The background shifts slightly to show different parts of the cityscape, including illuminated buildings and street signs."
+ 2. "Create a video where the character, dressed in historical attire, is seen holding an umbrella with a logo. The character should move closer to the camera while maintaining a steady pace, keeping the umbrella raised. The background remains consistent with a foggy, outdoor setting, but the focus shifts more towards the character as they approach. The lighting should emphasize the details of the costume and the umbrella, enhancing the dramatic effect."
+ 3. "Darken the scene while keeping the characters and setting unchanged, emphasizing a serious atmosphere."
+ IImportantly! These are just examples from a large training dataset of 20 mln videos.
+ Rewrite Prompt: "{prompt}" to get high-quality image to video generation from this image. Pay main attention to information about changes of objects.
+ Make prompt dynamic. Answer only with expanded prompt."""
+ PROMPT_TEMPLATE = {
+ "template": {
+ "video": (
+ "<|im_start|>system\nYou are a prompt engineer. Describe the video in detail.",
+ "Describe how the camera moves or shakes, describe the zoom and view angle, whether it follows the objects.",
+ "Describe the location of the video, main characters or objects and their action.",
+ "Describe the dynamism of the video and presented actions.",
+ "Name the visual style of the video: whether it is a professional footage, user generated content, some kind of animation, video game or scren content.",
+ "Describe the visual effects, postprocessing and transitions if they are presented in the video.",
+ "Pay attention to the order of key actions shown in the scene.<|im_end|>",
+ "<|im_start|>user\n{}<|im_end|>"
+ ),
+ "image2video": (
+ "<|im_start|>system\nYou are a prompt engineer. Your task is to create a highly detailed and effective video description based on a provided input image.",
+ "Describe how the camera moves or shakes, describe the zoom and view angle, whether it follows the objects.",
+ "Describe main characters actions.",
+ "Describe the dynamism of the video and presented actions.",
+ "Name the visual style of the video: whether it is a professional footage, user generated content, some kind of animation, video game or scren content.",
+ "Describe the visual effects, postprocessing and transitions if they are presented in the video.",
+ "Pay attention to the order of key actions shown in the scene.<|im_end|>",
+ "<|im_start|>user\n{}<|im_end|>"
+ ),
+ "image": (
+ "<|im_start|>system\nYou are a prompt engineer. Describe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>",
+ "<|im_start|>user\n{}<|im_end|>"
+ ),
+ "image_edit": (
+ "<|im_start|>system\nYou are a prompt engineer. Based on the provided source image (first image) and target image (second image), create an interesting text prompt that can be used together with the source image to create the target image:<|im_end|>",
+ "<|im_start|>user\n{}"
+ )
+ },
+ "crop_start": {
+ "video": 129,
+ "image": 41,
+ "image_edit": 55,
+ "image2video": 132
+ },
+ }
+
+ def __init__(self, conf, device, quantized_qwen=False, text_token_padding=True):
+ self._fast_loaded = False
+ checkpoint_path = conf.checkpoint_path
+ if os.path.isfile(checkpoint_path):
+ config_path = os.path.join(os.path.dirname(checkpoint_path), "config.json")
+ self.model = offload.fast_load_transformers_model(
+ checkpoint_path,
+ writable_tensors=True,
+ modelClass=Qwen2_5_VLForConditionalGeneration,
+ defaultConfigPath=config_path,
+ default_dtype=torch.bfloat16,
+ )
+ self.model = self.model.to(device)
+ self._fast_loaded = True
+ self.processor = AutoProcessor.from_pretrained(os.path.dirname(checkpoint_path), use_fast=True)
+ self.max_length = conf.max_length
+ self.text_token_padding = text_token_padding
+ self.model = freeze(self.model)
+ return
+
+ quantization_config = None
+ if quantized_qwen:
+ quantization_config = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_compute_dtype=torch.bfloat16,
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_quant_type="nf4"
+ )
+
+ self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+ checkpoint_path,
+ dtype=torch.bfloat16,
+ device_map=device,
+ quantization_config=quantization_config
+ )
+ self.model = freeze(self.model)
+ self.processor = AutoProcessor.from_pretrained(checkpoint_path, use_fast=True)
+ self.max_length = conf.max_length
+ self.text_token_padding = text_token_padding
+
+ def __call__(self, texts, images=None, type_of_content="video"):
+ prompt_template = "\n".join(self.PROMPT_TEMPLATE["template"][type_of_content])
+ crop_start = self.PROMPT_TEMPLATE["crop_start"][type_of_content]
+ full_texts = list(map(lambda x: prompt_template.format(x), texts))
+ if type_of_content == "image_edit":
+ if images is not None:
+ for i in range(len(images)):
+ image_tokens = ''.join(['<|vision_start|><|image_pad|><|vision_end|>']*len(images[i]))
+ full_texts[i] = full_texts[i] + image_tokens + "<|im_end|>"
+ images = [F.resize(i, (i.shape[-2] // 2, i.shape[-1] // 2)) for i in images]
+ max_length = (self.max_length + crop_start) if images is None else None
+ inputs = self.processor(
+ text=full_texts,
+ images=images,
+ truncation=True,
+ return_tensors="pt",
+ padding=True,
+ max_length=max_length
+ ).to(self.model.device)
+
+ with torch.no_grad():
+ embeds = self.model(**inputs, output_hidden_states=True)["hidden_states"][-1][:, crop_start:]
+ else:
+ max_length = self.max_length + crop_start
+ inputs = self.processor(
+ text=full_texts,
+ images=None,
+ videos=None,
+ max_length=max_length,
+ truncation=True,
+ return_tensors="pt",
+ padding="max_length",
+ ).to(self.model.device)
+
+ with torch.no_grad():
+ embeds = self.model(
+ input_ids=inputs["input_ids"],
+ return_dict=True,
+ output_hidden_states=True,
+ )["hidden_states"][-1][:, crop_start:]
+ attention_mask = inputs["attention_mask"][:, crop_start:]
+ if self.text_token_padding:
+ seq_length = embeds.shape[1]
+ cu_seqlens = torch.tensor([0, seq_length], dtype=torch.int32)
+ else:
+ embeds = embeds[attention_mask.bool()].unsqueeze(0)
+ cu_seqlens = torch.cumsum(attention_mask.sum(1), dim=0)
+ cu_seqlens = torch.cat([torch.zeros_like(cu_seqlens)[:1], cu_seqlens]).to(
+ dtype=torch.int32
+ )
+ attention_mask = None
+ return embeds, cu_seqlens, attention_mask
+
+ def expand_text_prompt(self, prompt, image, device="cuda"):
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image",
+ "image": image
+ },
+ {
+ "type": "text",
+ "text": self.INSTRUCTION_I2V_EXPAND.format(prompt=prompt),
+ },
+ ],
+ }
+ ]
+
+ text = self.processor.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True
+ )
+ inputs = self.processor(
+ text=[text],
+ images=[image],
+ videos=None,
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to(device)
+ generated_ids = self.model.generate(
+ **inputs, max_new_tokens=256
+ )
+ generated_ids_trimmed = [
+ out_ids[len(in_ids):]
+ for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ ]
+ output_text = self.processor.batch_decode(
+ generated_ids_trimmed,
+ skip_special_tokens=True,
+ clean_up_tokenization_spaces=False,
+ )
+ return output_text[0]
+
+
+class Kandinsky5TextEmbedder:
+ def __init__(self, conf, device="cpu", quantized_qwen=False, text_token_padding=False):
+ self.embedder = Qwen2_5_VLTextEmbedder(conf.qwen, device, quantized_qwen, text_token_padding)
+ self.clip_embedder = ClipTextEmbedder(conf.clip, device)
+ self.conf = conf
+
+ def encode(self, texts, images=None, type_of_content="image"):
+ text_embeds, cu_seqlens, attention_mask = self.embedder(texts, images=images, type_of_content=type_of_content)
+ pooled_embed = self.clip_embedder(texts)
+ if attention_mask is not None:
+ attention_mask = attention_mask.to(torch.bool)
+ return {"text_embeds": text_embeds, "pooled_embed": pooled_embed}, cu_seqlens, attention_mask
+
+ def to(self, device):
+ self.embedder.model = self.embedder.model.to(device)
+ self.clip_embedder.model = self.clip_embedder.model.to(device)
+ return self
+
+
+def get_text_embedder(conf, device="cpu", quantized_qwen=False, text_token_padding=True):
+ return Kandinsky5TextEmbedder(conf, device, quantized_qwen, text_token_padding)
diff --git a/models/kandinsky5/kandinsky/models/utils.py b/models/kandinsky5/kandinsky/models/utils.py
new file mode 100644
index 0000000..fd15c1d
--- /dev/null
+++ b/models/kandinsky5/kandinsky/models/utils.py
@@ -0,0 +1,169 @@
+import math
+
+import torch
+
+from torch import Tensor, IntTensor, BoolTensor
+try:
+ from torch.nn.attention.flex_attention import BlockMask, _mask_mod_signature
+except Exception:
+ BlockMask = None
+ _mask_mod_signature = None
+
+from einops import rearrange
+
+
+def exist(item):
+ return item is not None
+
+
+def freeze(model):
+ for p in model.parameters():
+ p.requires_grad = False
+ return model
+
+
[email protected](device_type="cuda", enabled=False)
+def get_freqs(dim, max_period=10000.0):
+ freqs = torch.exp(
+ -math.log(max_period)
+ * torch.arange(start=0, end=dim, dtype=torch.float32)
+ / dim
+ )
+ return freqs
+
+
+def fractal_flatten(x, rope, shape, block_mask=False):
+ if block_mask:
+ pixel_size = 8
+ x = local_patching(x, shape, (1, pixel_size, pixel_size), dim=0)
+ rope = local_patching(rope, shape, (1, pixel_size, pixel_size), dim=0)
+ x = x.flatten(0, 1)
+ rope = rope.flatten(0, 1)
+ else:
+ x = x.flatten(0, 2)
+ rope = rope.flatten(0, 2)
+ return x, rope
+
+
+def fractal_unflatten(x, shape, block_mask=False):
+ if block_mask:
+ pixel_size = 8
+ x = x.reshape(-1, pixel_size**2, x.shape[-1])
+ x = local_merge(x, shape, (1, pixel_size, pixel_size), dim=0)
+ else:
+ x = x.reshape(*shape, x.shape[-1])
+ return x
+
+
+def local_patching(x, shape, group_size, dim=0):
+ duration, height, width = shape
+ g1, g2, g3 = group_size
+ x = x.reshape(
+ *x.shape[:dim],
+ duration // g1,
+ g1,
+ height // g2,
+ g2,
+ width // g3,
+ g3,
+ *x.shape[dim + 3 :]
+ )
+ x = x.permute(
+ *range(len(x.shape[:dim])),
+ dim,
+ dim + 2,
+ dim + 4,
+ dim + 1,
+ dim + 3,
+ dim + 5,
+ *range(dim + 6, len(x.shape))
+ )
+ x = x.flatten(dim, dim + 2).flatten(dim + 1, dim + 3)
+ return x
+
+
+def local_merge(x, shape, group_size, dim=0):
+ duration, height, width = shape
+ g1, g2, g3 = group_size
+ x = x.reshape(
+ *x.shape[:dim],
+ duration // g1,
+ height // g2,
+ width // g3,
+ g1,
+ g2,
+ g3,
+ *x.shape[dim + 2 :]
+ )
+ x = x.permute(
+ *range(len(x.shape[:dim])),
+ dim,
+ dim + 3,
+ dim + 1,
+ dim + 4,
+ dim + 2,
+ dim + 5,
+ *range(dim + 6, len(x.shape))
+ )
+ x = x.flatten(dim, dim + 1).flatten(dim + 1, dim + 2).flatten(dim + 2, dim + 3)
+ return x
+
+
+def fast_sta_nabla(
+ T: int, H: int, W: int, wT: int = 3, wH: int = 3, wW: int = 3, device="cuda"
+) -> Tensor:
+ l = torch.Tensor([T, H, W]).amax()
+ r = torch.arange(0, l, 1, dtype=torch.int16, device=device)
+ mat = (r.unsqueeze(1) - r.unsqueeze(0)).abs()
+ sta_t, sta_h, sta_w = (
+ mat[:T, :T].flatten(),
+ mat[:H, :H].flatten(),
+ mat[:W, :W].flatten(),
+ )
+ sta_t = sta_t <= wT // 2
+ sta_h = sta_h <= wH // 2
+ sta_w = sta_w <= wW // 2
+ sta_hw = (
+ (sta_h.unsqueeze(1) * sta_w.unsqueeze(0))
+ .reshape(H, H, W, W)
+ .transpose(1, 2)
+ .flatten()
+ )
+ sta = (
+ (sta_t.unsqueeze(1) * sta_hw.unsqueeze(0))
+ .reshape(T, T, H * W, H * W)
+ .transpose(1, 2)
+ )
+ return sta.reshape(T * H * W, T * H * W)
+
+
+def nablaT_v2(
+ q: Tensor,
+ k: Tensor,
+ sta: Tensor,
+ thr: float = 0.9,
+) -> BlockMask:
+ if BlockMask is None:
+ raise RuntimeError("flex_attention BlockMask is not available.")
+ # Map estimation
+ B, h, S, D = q.shape
+ s1 = S // 64
+ qa = q.reshape(B, h, s1, 64, D).mean(-2)
+ ka = k.reshape(B, h, s1, 64, D).mean(-2).transpose(-2, -1)
+ map = qa @ ka
+
+ map = torch.softmax(map / math.sqrt(D), dim=-1)
+ # Map binarization
+ vals, inds = map.sort(-1)
+ cvals = vals.cumsum_(-1)
+ mask = (cvals >= 1 - thr).int()
+ mask = mask.gather(-1, inds.argsort(-1))
+
+ mask = torch.logical_or(mask, sta)
+
+ # BlockMask creation
+ kv_nb = mask.sum(-1).to(torch.int32)
+ kv_inds = mask.argsort(dim=-1, descending=True).to(torch.int32)
+ return BlockMask.from_kv_blocks(
+ torch.zeros_like(kv_nb), kv_inds, kv_nb, kv_inds, BLOCK_SIZE=64, mask_mod=None
+ )
diff --git a/models/kandinsky5/kandinsky/models/vae.py b/models/kandinsky5/kandinsky/models/vae.py
new file mode 100644
index 0000000..57deadb
--- /dev/null
+++ b/models/kandinsky5/kandinsky/models/vae.py
@@ -0,0 +1,1310 @@
+import os
+import json
+from math import sqrt, floor, ceil
+from typing import Optional, Tuple, Union, List
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.utils.accelerate_utils import apply_forward_hook
+from diffusers.models.activations import get_activation
+from diffusers.models.attention_processor import Attention
+from diffusers.models.modeling_outputs import AutoencoderKLOutput
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.models.autoencoders.vae import (
+ DecoderOutput,
+ DiagonalGaussianDistribution,
+)
+
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+os.environ["TORCHINDUCTOR_FX_GRAPH_CACHE"] = "1"
+torch.backends.cudnn.allow_tf32 = True
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.benchmark = True
+
+
+def prepare_causal_attention_mask(
+ f: int, s: int, dtype: torch.dtype, device: torch.device, b: int
+) -> torch.Tensor:
+ return (
+ torch.ones((f, f), dtype=dtype, device=device)
+ .tril_()
+ .log_()
+ .repeat_interleave(s, dim=0)
+ .repeat_interleave(s, dim=1)
+ .unsqueeze(0)
+ .expand(b, -1, -1)
+ .contiguous()
+ )
+
+
+class HunyuanVideoCausalConv3d(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: Union[int, Tuple[int, int, int]] = 3,
+ stride: Union[int, Tuple[int, int, int]] = 1,
+ padding: Union[int, Tuple[int, int, int]] = 0,
+ dilation: Union[int, Tuple[int, int, int]] = 1,
+ bias: bool = True,
+ pad_mode: str = "replicate",
+ ) -> None:
+ super().__init__()
+
+ kernel_size = (
+ (kernel_size, kernel_size, kernel_size)
+ if isinstance(kernel_size, int)
+ else kernel_size
+ )
+
+ self.pad_mode = pad_mode
+ self.time_causal_padding = (
+ kernel_size[0] // 2,
+ kernel_size[0] // 2,
+ kernel_size[1] // 2,
+ kernel_size[1] // 2,
+ kernel_size[2] - 1,
+ 0,
+ )
+
+ self.conv = nn.Conv3d(
+ in_channels, out_channels, kernel_size, stride, padding, dilation, bias=bias
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = F.pad(
+ hidden_states, self.time_causal_padding, mode=self.pad_mode
+ )
+ return self.conv(hidden_states)
+
+
+class HunyuanVideoUpsampleCausal3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: Optional[int] = None,
+ kernel_size: int = 3,
+ stride: int = 1,
+ bias: bool = True,
+ upsample_factor: Tuple[float, float, float] = (2, 2, 2),
+ ) -> None:
+ super().__init__()
+
+ out_channels = out_channels or in_channels
+ self.upsample_factor = upsample_factor
+
+ self.conv = HunyuanVideoCausalConv3d(
+ in_channels, out_channels, kernel_size, stride, bias=bias
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ num_frames = hidden_states.size(2)
+ dtp = hidden_states.dtype
+ first_frame, other_frames = hidden_states.split((1, num_frames - 1), dim=2)
+ first_frame = F.interpolate(
+ first_frame.squeeze(2),
+ scale_factor=self.upsample_factor[1:],
+ mode="nearest",
+ ).unsqueeze(2).to(dtp) #force cast
+
+ if num_frames > 1:
+ other_frames = other_frames.contiguous()
+ other_frames = F.interpolate(
+ other_frames, scale_factor=self.upsample_factor, mode="nearest"
+ ).to(dtp) # force cast
+ hidden_states = torch.cat((first_frame, other_frames), dim=2)
+ del first_frame
+ del other_frames
+ else:
+ hidden_states = first_frame
+
+ hidden_states = self.conv(hidden_states)
+ return hidden_states
+
+
+class HunyuanVideoDownsampleCausal3D(nn.Module):
+ def __init__(
+ self,
+ channels: int,
+ out_channels: Optional[int] = None,
+ padding: int = 1,
+ kernel_size: int = 3,
+ bias: bool = True,
+ stride=2,
+ ) -> None:
+ super().__init__()
+ out_channels = out_channels or channels
+
+ self.conv = HunyuanVideoCausalConv3d(
+ channels, out_channels, kernel_size, stride, padding, bias=bias
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = self.conv(hidden_states)
+ return hidden_states
+
+
+class HunyuanVideoResnetBlockCausal3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: Optional[int] = None,
+ dropout: float = 0.0,
+ groups: int = 32,
+ eps: float = 1e-6,
+ non_linearity: str = "swish",
+ ) -> None:
+ super().__init__()
+ out_channels = out_channels or in_channels
+
+ self.nonlinearity = get_activation(non_linearity)
+
+ self.norm1 = nn.GroupNorm(groups, in_channels, eps=eps, affine=True)
+ self.conv1 = HunyuanVideoCausalConv3d(in_channels, out_channels, 3, 1, 0)
+
+ self.norm2 = nn.GroupNorm(groups, out_channels, eps=eps, affine=True)
+ self.dropout = nn.Dropout(dropout)
+ self.conv2 = HunyuanVideoCausalConv3d(out_channels, out_channels, 3, 1, 0)
+
+ self.conv_shortcut = None
+ if in_channels != out_channels:
+ self.conv_shortcut = HunyuanVideoCausalConv3d(
+ in_channels, out_channels, 1, 1, 0
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ dtp = hidden_states.dtype
+ hidden_states = hidden_states.contiguous()
+ residual = hidden_states
+
+ hidden_states = self.norm1(hidden_states).to(dtp) #force cast
+ hidden_states = self.nonlinearity(hidden_states)
+ hidden_states = self.conv1(hidden_states)
+
+ hidden_states = self.norm2(hidden_states).to(dtp) #force cast
+ hidden_states = self.nonlinearity(hidden_states)
+ hidden_states = self.dropout(hidden_states)
+ hidden_states = self.conv2(hidden_states)
+
+ if self.conv_shortcut is not None:
+ residual = self.conv_shortcut(residual)
+
+ hidden_states = hidden_states + residual
+ return hidden_states
+
+
+class HunyuanVideoMidBlock3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ add_attention: bool = True,
+ attention_head_dim: int = 1,
+ ) -> None:
+ super().__init__()
+ resnet_groups = (
+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
+ )
+ self.add_attention = add_attention
+
+ # There is always at least one resnet
+ resnets = [
+ HunyuanVideoResnetBlockCausal3D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ non_linearity=resnet_act_fn,
+ )
+ ]
+ attentions = []
+
+ for _ in range(num_layers):
+ if self.add_attention:
+ attentions.append(
+ Attention(
+ in_channels,
+ heads=in_channels // attention_head_dim,
+ dim_head=attention_head_dim,
+ eps=resnet_eps,
+ norm_num_groups=resnet_groups,
+ residual_connection=True,
+ bias=True,
+ upcast_softmax=True,
+ _from_deprecated_attn_block=True,
+ )
+ )
+ else:
+ attentions.append(None)
+
+ resnets.append(
+ HunyuanVideoResnetBlockCausal3D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ non_linearity=resnet_act_fn,
+ )
+ )
+
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = self.resnets[0](hidden_states)
+
+ for attn, resnet in zip(self.attentions, self.resnets[1:]):
+ if attn is not None:
+ batch_size, _, num_frames, height, width = hidden_states.shape
+ hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3)
+ mask = prepare_causal_attention_mask(
+ num_frames,
+ height * width,
+ hidden_states.dtype,
+ hidden_states.device,
+ batch_size,
+ )
+ hidden_states = attn(hidden_states, attention_mask=mask)
+ hidden_states = hidden_states.unflatten(
+ 1, (num_frames, height, width)
+ ).permute(0, 4, 1, 2, 3)
+
+ hidden_states = resnet(hidden_states)
+
+ return hidden_states
+
+
+class HunyuanVideoDownBlock3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ add_downsample: bool = True,
+ downsample_stride: int = 2,
+ downsample_padding: int = 1,
+ ) -> None:
+ super().__init__()
+ resnets = []
+
+ for i in range(num_layers):
+ in_channels = in_channels if i == 0 else out_channels
+ resnets.append(
+ HunyuanVideoResnetBlockCausal3D(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ non_linearity=resnet_act_fn,
+ )
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_downsample:
+ self.downsamplers = nn.ModuleList(
+ [
+ HunyuanVideoDownsampleCausal3D(
+ out_channels,
+ out_channels=out_channels,
+ padding=downsample_padding,
+ stride=downsample_stride,
+ )
+ ]
+ )
+ else:
+ self.downsamplers = None
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ for resnet in self.resnets:
+ hidden_states = resnet(hidden_states)
+
+ if self.downsamplers is not None:
+ for downsampler in self.downsamplers:
+ hidden_states = downsampler(hidden_states)
+
+ return hidden_states
+
+
+class HunyuanVideoUpBlock3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ add_upsample: bool = True,
+ upsample_scale_factor: Tuple[int, int, int] = (2, 2, 2),
+ ) -> None:
+ super().__init__()
+ resnets = []
+
+ for i in range(num_layers):
+ input_channels = in_channels if i == 0 else out_channels
+
+ resnets.append(
+ HunyuanVideoResnetBlockCausal3D(
+ in_channels=input_channels,
+ out_channels=out_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ non_linearity=resnet_act_fn,
+ )
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_upsample:
+ self.upsamplers = nn.ModuleList(
+ [
+ HunyuanVideoUpsampleCausal3D(
+ out_channels,
+ out_channels=out_channels,
+ upsample_factor=upsample_scale_factor,
+ )
+ ]
+ )
+ else:
+ self.upsamplers = None
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ for resnet in self.resnets:
+ hidden_states = resnet(hidden_states)
+
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(hidden_states)
+
+ return hidden_states
+
+
+class HunyuanVideoEncoder3D(nn.Module):
+ r"""
+ Causal encoder for 3D video-like data introduced
+ in [Hunyuan Video](https://huggingface.co/papers/2412.03603).
+ """
+
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 3,
+ down_block_types: Tuple[str, ...] = (
+ "HunyuanVideoDownBlock3D",
+ "HunyuanVideoDownBlock3D",
+ "HunyuanVideoDownBlock3D",
+ "HunyuanVideoDownBlock3D",
+ ),
+ block_out_channels: Tuple[int, ...] = (128, 256, 512, 512),
+ layers_per_block: int = 2,
+ norm_num_groups: int = 32,
+ act_fn: str = "silu",
+ double_z: bool = True,
+ mid_block_add_attention=True,
+ temporal_compression_ratio: int = 4,
+ spatial_compression_ratio: int = 8,
+ ) -> None:
+ super().__init__()
+
+ self.conv_in = HunyuanVideoCausalConv3d(
+ in_channels, block_out_channels[0], kernel_size=3, stride=1
+ )
+ self.mid_block = None
+ self.down_blocks = nn.ModuleList([])
+
+ output_channel = block_out_channels[0]
+ for i, down_block_type in enumerate(down_block_types):
+ if down_block_type != "HunyuanVideoDownBlock3D":
+ raise ValueError(f"Unsupported down_block_type: {down_block_type}")
+
+ input_channel = output_channel
+ output_channel = block_out_channels[i]
+ is_final_block = i == len(block_out_channels) - 1
+ num_spatial_downsample_layers = int(np.log2(spatial_compression_ratio))
+ num_time_downsample_layers = int(np.log2(temporal_compression_ratio))
+
+ if temporal_compression_ratio == 4:
+ add_spatial_downsample = bool(i < num_spatial_downsample_layers)
+ add_time_downsample = bool(
+ i >= (len(block_out_channels) - 1 - num_time_downsample_layers)
+ and not is_final_block
+ )
+ elif temporal_compression_ratio == 8:
+ add_spatial_downsample = bool(i < num_spatial_downsample_layers)
+ add_time_downsample = bool(i < num_time_downsample_layers)
+ else:
+ raise ValueError(
+ f"Unsupported time_compression_ratio: {temporal_compression_ratio}"
+ )
+
+ downsample_stride_HW = (2, 2) if add_spatial_downsample else (1, 1)
+ downsample_stride_T = (2,) if add_time_downsample else (1,)
+ downsample_stride = tuple(downsample_stride_T + downsample_stride_HW)
+
+ down_block = HunyuanVideoDownBlock3D(
+ num_layers=layers_per_block,
+ in_channels=input_channel,
+ out_channels=output_channel,
+ add_downsample=bool(add_spatial_downsample or add_time_downsample),
+ resnet_eps=1e-6,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ downsample_stride=downsample_stride,
+ downsample_padding=0,
+ )
+
+ self.down_blocks.append(down_block)
+
+ self.mid_block = HunyuanVideoMidBlock3D(
+ in_channels=block_out_channels[-1],
+ resnet_eps=1e-6,
+ resnet_act_fn=act_fn,
+ attention_head_dim=block_out_channels[-1],
+ resnet_groups=norm_num_groups,
+ add_attention=mid_block_add_attention,
+ )
+
+ self.conv_norm_out = nn.GroupNorm(
+ num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6
+ )
+ self.conv_act = nn.SiLU()
+
+ conv_out_channels = 2 * out_channels if double_z else out_channels
+ self.conv_out = HunyuanVideoCausalConv3d(
+ block_out_channels[-1], conv_out_channels, kernel_size=3
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = self.conv_in(hidden_states)
+
+ for down_block in self.down_blocks:
+ hidden_states = down_block(hidden_states)
+
+ hidden_states = self.mid_block(hidden_states)
+
+ hidden_states = self.conv_norm_out(hidden_states)
+ hidden_states = self.conv_act(hidden_states)
+ hidden_states = self.conv_out(hidden_states)
+
+ return hidden_states
+
+
+class HunyuanVideoDecoder3D(nn.Module):
+ r"""
+ Causal decoder for 3D video-like data introduced
+ in [Hunyuan Video](https://huggingface.co/papers/2412.03603).
+ """
+
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 3,
+ up_block_types: Tuple[str, ...] = (
+ "HunyuanVideoUpBlock3D",
+ "HunyuanVideoUpBlock3D",
+ "HunyuanVideoUpBlock3D",
+ "HunyuanVideoUpBlock3D",
+ ),
+ block_out_channels: Tuple[int, ...] = (128, 256, 512, 512),
+ layers_per_block: int = 2,
+ norm_num_groups: int = 32,
+ act_fn: str = "silu",
+ mid_block_add_attention=True,
+ time_compression_ratio: int = 4,
+ spatial_compression_ratio: int = 8,
+ ):
+ super().__init__()
+ self.layers_per_block = layers_per_block
+
+ self.conv_in = HunyuanVideoCausalConv3d(
+ in_channels, block_out_channels[-1], kernel_size=3, stride=1
+ )
+ self.up_blocks = nn.ModuleList([])
+
+ # mid
+ self.mid_block = HunyuanVideoMidBlock3D(
+ in_channels=block_out_channels[-1],
+ resnet_eps=1e-6,
+ resnet_act_fn=act_fn,
+ attention_head_dim=block_out_channels[-1],
+ resnet_groups=norm_num_groups,
+ add_attention=mid_block_add_attention,
+ )
+
+ # up
+ reversed_block_out_channels = list(reversed(block_out_channels))
+ output_channel = reversed_block_out_channels[0]
+ for i, up_block_type in enumerate(up_block_types):
+ if up_block_type != "HunyuanVideoUpBlock3D":
+ raise ValueError(f"Unsupported up_block_type: {up_block_type}")
+
+ prev_output_channel = output_channel
+ output_channel = reversed_block_out_channels[i]
+ is_final_block = i == len(block_out_channels) - 1
+ num_spatial_upsample_layers = int(np.log2(spatial_compression_ratio))
+ num_time_upsample_layers = int(np.log2(time_compression_ratio))
+
+ if time_compression_ratio == 4:
+ add_spatial_upsample = bool(i < num_spatial_upsample_layers)
+ add_time_upsample = bool(
+ i >= len(block_out_channels) - 1 - num_time_upsample_layers
+ and not is_final_block
+ )
+ else:
+ raise ValueError(
+ f"Unsupported time_compression_ratio: {time_compression_ratio}"
+ )
+
+ upsample_scale_factor_HW = (2, 2) if add_spatial_upsample else (1, 1)
+ upsample_scale_factor_T = (2,) if add_time_upsample else (1,)
+ upsample_scale_factor = tuple(
+ upsample_scale_factor_T + upsample_scale_factor_HW
+ )
+
+ up_block = HunyuanVideoUpBlock3D(
+ num_layers=self.layers_per_block + 1,
+ in_channels=prev_output_channel,
+ out_channels=output_channel,
+ add_upsample=bool(add_spatial_upsample or add_time_upsample),
+ upsample_scale_factor=upsample_scale_factor,
+ resnet_eps=1e-6,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ )
+
+ self.up_blocks.append(up_block)
+ prev_output_channel = output_channel
+
+ # out
+ self.conv_norm_out = nn.GroupNorm(
+ num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6
+ )
+ self.conv_act = nn.SiLU()
+ self.conv_out = HunyuanVideoCausalConv3d(
+ block_out_channels[0], out_channels, kernel_size=3
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ dtp = hidden_states.dtype
+ hidden_states = self.conv_in(hidden_states)
+
+ hidden_states = self.mid_block(hidden_states)
+
+ for up_block in self.up_blocks:
+ hidden_states = up_block(hidden_states)
+
+ hidden_states = self.conv_norm_out(hidden_states)
+ hidden_states = self.conv_act(hidden_states).to(dtp) # force cast
+ hidden_states = self.conv_out(hidden_states)
+
+ return hidden_states
+
+
+class AutoencoderKLHunyuanVideo(ModelMixin, ConfigMixin):
+ r"""
+ A VAE model with KL loss for encoding videos into latents
+ and decoding latent representations into videos.
+ Introduced in [HunyuanVideo](https://huggingface.co/papers/2412.03603).
+
+ This model inherits from [`ModelMixin`]. Check the superclass
+ documentation for it's generic methods implemented
+ for all models (such as downloading or saving).
+ """
+
+ @register_to_config
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 3,
+ latent_channels: int = 16,
+ down_block_types: Tuple[str, ...] = (
+ "HunyuanVideoDownBlock3D",
+ "HunyuanVideoDownBlock3D",
+ "HunyuanVideoDownBlock3D",
+ "HunyuanVideoDownBlock3D",
+ ),
+ up_block_types: Tuple[str, ...] = (
+ "HunyuanVideoUpBlock3D",
+ "HunyuanVideoUpBlock3D",
+ "HunyuanVideoUpBlock3D",
+ "HunyuanVideoUpBlock3D",
+ ),
+ block_out_channels: Tuple[int] = (128, 256, 512, 512),
+ layers_per_block: int = 2,
+ act_fn: str = "silu",
+ norm_num_groups: int = 32,
+ scaling_factor: float = 0.476986,
+ spatial_compression_ratio: int = 8,
+ temporal_compression_ratio: int = 4,
+ mid_block_add_attention: bool = True,
+ ) -> None:
+ super().__init__()
+
+ self.time_compression_ratio = temporal_compression_ratio
+
+ self.encoder = HunyuanVideoEncoder3D(
+ in_channels=in_channels,
+ out_channels=latent_channels,
+ down_block_types=down_block_types,
+ block_out_channels=block_out_channels,
+ layers_per_block=layers_per_block,
+ norm_num_groups=norm_num_groups,
+ act_fn=act_fn,
+ double_z=True,
+ mid_block_add_attention=mid_block_add_attention,
+ temporal_compression_ratio=temporal_compression_ratio,
+ spatial_compression_ratio=spatial_compression_ratio,
+ )
+
+ self.decoder = HunyuanVideoDecoder3D(
+ in_channels=latent_channels,
+ out_channels=out_channels,
+ up_block_types=up_block_types,
+ block_out_channels=block_out_channels,
+ layers_per_block=layers_per_block,
+ norm_num_groups=norm_num_groups,
+ act_fn=act_fn,
+ time_compression_ratio=temporal_compression_ratio,
+ spatial_compression_ratio=spatial_compression_ratio,
+ mid_block_add_attention=mid_block_add_attention,
+ )
+
+ self.quant_conv = nn.Conv3d(
+ 2 * latent_channels, 2 * latent_channels, kernel_size=1
+ )
+ self.post_quant_conv = nn.Conv3d(
+ latent_channels, latent_channels, kernel_size=1
+ )
+
+ self.spatial_compression_ratio = spatial_compression_ratio
+ self.temporal_compression_ratio = temporal_compression_ratio
+
+ self.use_slicing = False
+
+ self.use_tiling = True
+
+ self.use_framewise_encoding = True
+ self.use_framewise_decoding = True
+
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_sample_min_num_frames = 16
+
+ self.tile_sample_stride_height = 192
+ self.tile_sample_stride_width = 192
+ self.tile_sample_stride_num_frames = 12
+
+ self.tile_overlap_factor = 0.25
+ self._forced_tile_config = None
+ self.tile_size = None
+
+ def _encode(self, x: torch.Tensor) -> torch.Tensor:
+ _, _, num_frames, height, width = x.shape
+
+ if self.use_framewise_decoding and num_frames > (
+ self.tile_sample_min_num_frames + 1
+ ):
+ return self._temporal_tiled_encode(x)
+
+ if self.use_tiling and (
+ width > self.tile_sample_min_width or height > self.tile_sample_min_height
+ ):
+ return self.tiled_encode(x)
+
+ x = self.encoder(x)
+ enc = self.quant_conv(x)
+ return enc
+
+ @apply_forward_hook
+ def encode(
+ self, x: torch.Tensor, opt_tiling: bool = True, return_dict: bool = True
+ ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]:
+ r"""
+ Encode a batch of images into latents.
+
+ Args:
+ x (`torch.Tensor`): Input batch of images.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`]
+ instead of a plain tuple.
+
+ Returns:
+ The latent representations of the encoded videos. If `return_dict` is True, a
+ [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned,
+ otherwise a plain `tuple` is returned.
+ """
+ if opt_tiling:
+ if self._forced_tile_config is not None:
+ tile_size, tile_stride = self._forced_tile_config
+ else:
+ tile_size, tile_stride = self.get_enc_optimal_tiling(x.shape)
+ else:
+ b, _, f, h, w = x.shape
+ tile_size, tile_stride = (b, f, h, w), (f, h, w)
+ if tile_size != self.tile_size:
+ self.tile_size = tile_size
+ self.apply_tiling(tile_size, tile_stride)
+
+ h = self._encode(x)
+
+ posterior = DiagonalGaussianDistribution(h)
+
+ if not return_dict:
+ return (posterior,)
+ return AutoencoderKLOutput(latent_dist=posterior)
+
+ def _decode(
+ self, z: torch.Tensor, return_dict: bool = True
+ ) -> Union[DecoderOutput, torch.Tensor]:
+ _, _, num_frames, height, width = z.shape
+ tile_latent_min_height = (
+ self.tile_sample_min_height // self.spatial_compression_ratio
+ )
+ tile_latent_min_width = (
+ self.tile_sample_min_width // self.spatial_compression_ratio
+ )
+ tile_latent_min_num_frames = (
+ self.tile_sample_min_num_frames // self.temporal_compression_ratio
+ )
+
+ if self.use_framewise_decoding and num_frames > (
+ tile_latent_min_num_frames + 1
+ ):
+ return self._temporal_tiled_decode(z, return_dict=return_dict)
+
+ if self.use_tiling and (
+ width > tile_latent_min_width or height > tile_latent_min_height
+ ):
+ return self.tiled_decode(z, return_dict=return_dict)
+
+ z = self.post_quant_conv(z)
+ dec = self.decoder(z)
+
+ if not return_dict:
+ return (dec,)
+
+ return DecoderOutput(sample=dec)
+
+ @apply_forward_hook
+ def decode(
+ self, z: torch.Tensor, return_dict: bool = True
+ ) -> Union[DecoderOutput, torch.Tensor]:
+ r"""
+ Decode a batch of images.
+
+ Args:
+ z (`torch.Tensor`): Input batch of latent vectors.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.vae.DecoderOutput`] or `tuple`:
+ If return_dict is True, a [`~models.vae.DecoderOutput`] is returned,
+ otherwise a plain `tuple` is returned.
+ """
+ if self._forced_tile_config is not None:
+ tile_size, tile_stride = self._forced_tile_config
+ else:
+ tile_size, tile_stride = self.get_dec_optimal_tiling(z.shape)
+ if tile_size != self.tile_size:
+ self.tile_size = tile_size
+ self.apply_tiling(tile_size, tile_stride)
+
+ decoded = self._decode(z).sample
+
+ if not return_dict:
+ return (decoded,)
+
+ return DecoderOutput(sample=decoded)
+
+ def blend_v(
+ self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
+ ) -> torch.Tensor:
+ blend_extent = min(a.shape[-2], b.shape[-2], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (
+ 1 - y / blend_extent
+ ) + b[:, :, :, y, :] * (y / blend_extent)
+ return b
+
+ def blend_h(
+ self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
+ ) -> torch.Tensor:
+ blend_extent = min(a.shape[-1], b.shape[-1], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (
+ 1 - x / blend_extent
+ ) + b[:, :, :, :, x] * (x / blend_extent)
+ return b
+
+ def blend_t(
+ self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
+ ) -> torch.Tensor:
+ blend_extent = min(a.shape[-3], b.shape[-3], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (
+ 1 - x / blend_extent
+ ) + b[:, :, x, :, :] * (x / blend_extent)
+ return b
+
+ def tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput:
+ r"""Encode a batch of images using a tiled encoder.
+
+ Args:
+ x (`torch.Tensor`): Input batch of videos.
+
+ Returns:
+ `torch.Tensor`:
+ The latent representation of the encoded videos.
+ """
+ _, _, _, height, width = x.shape
+ latent_height = height // self.spatial_compression_ratio
+ latent_width = width // self.spatial_compression_ratio
+
+ tile_latent_min_height = (
+ self.tile_sample_min_height // self.spatial_compression_ratio
+ )
+ tile_latent_min_width = (
+ self.tile_sample_min_width // self.spatial_compression_ratio
+ )
+ tile_latent_stride_height = (
+ self.tile_sample_stride_height // self.spatial_compression_ratio
+ )
+ tile_latent_stride_width = (
+ self.tile_sample_stride_width // self.spatial_compression_ratio
+ )
+
+ blend_height = tile_latent_min_height - tile_latent_stride_height
+ blend_width = tile_latent_min_width - tile_latent_stride_width
+
+ rows = []
+ for i in range(
+ 0, height - self.tile_sample_min_height + 1, self.tile_sample_stride_height
+ ):
+ row = []
+ for j in range(
+ 0, width - self.tile_sample_min_width + 1, self.tile_sample_stride_width
+ ):
+ tile = x[
+ :,
+ :,
+ :,
+ i : i + self.tile_sample_min_height,
+ j : j + self.tile_sample_min_width,
+ ]
+ tile = self.encoder(tile).clone()
+ tile = self.quant_conv(tile)
+ row.append(tile)
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_width)
+ height_lim = (
+ tile_latent_min_height
+ if i == len(rows) - 1
+ else tile_latent_stride_height
+ )
+ width_lim = (
+ tile_latent_min_width
+ if j == len(row) - 1
+ else tile_latent_stride_width
+ )
+ result_row.append(tile[:, :, :, :height_lim, :width_lim])
+ result_rows.append(torch.cat(result_row, dim=4))
+
+ enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width]
+ return enc
+
+ def tiled_decode(
+ self, z: torch.Tensor, return_dict: bool = True
+ ) -> Union[DecoderOutput, torch.Tensor]:
+ r"""
+ Decode a batch of images using a tiled decoder.
+
+ Args:
+ z (`torch.Tensor`): Input batch of latent vectors.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.vae.DecoderOutput`] or `tuple`:
+ If return_dict is True, a [`~models.vae.DecoderOutput`] is returned,
+ otherwise a plain `tuple` is returned.
+ """
+
+ _, _, _, height, width = z.shape
+ sample_height = height * self.spatial_compression_ratio
+ sample_width = width * self.spatial_compression_ratio
+
+ tile_latent_min_height = (
+ self.tile_sample_min_height // self.spatial_compression_ratio
+ )
+ tile_latent_min_width = (
+ self.tile_sample_min_width // self.spatial_compression_ratio
+ )
+ tile_latent_stride_height = (
+ self.tile_sample_stride_height // self.spatial_compression_ratio
+ )
+ tile_latent_stride_width = (
+ self.tile_sample_stride_width // self.spatial_compression_ratio
+ )
+
+ blend_height = self.tile_sample_min_height - self.tile_sample_stride_height
+ blend_width = self.tile_sample_min_width - self.tile_sample_stride_width
+
+ rows = []
+ for i in range(
+ 0, height - tile_latent_min_height + 1, tile_latent_stride_height
+ ):
+ row = []
+ for j in range(
+ 0, width - tile_latent_min_width + 1, tile_latent_stride_width
+ ):
+ tile = z[
+ :,
+ :,
+ :,
+ i : i + tile_latent_min_height,
+ j : j + tile_latent_min_width,
+ ]
+ tile = self.post_quant_conv(tile)
+ decoded = self.decoder(tile).clone()
+ row.append(decoded)
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_width)
+ height_lim = (
+ self.tile_sample_min_height
+ if i == len(rows) - 1
+ else self.tile_sample_stride_height
+ )
+ width_lim = (
+ self.tile_sample_min_width
+ if j == len(row) - 1
+ else self.tile_sample_stride_width
+ )
+ result_row.append(tile[:, :, :, :height_lim, :width_lim])
+ result_rows.append(torch.cat(result_row, dim=-1))
+
+ dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width]
+
+ if not return_dict:
+ return (dec,)
+ return DecoderOutput(sample=dec)
+
+ def _temporal_tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput:
+ _, _, num_frames, height, width = x.shape
+ latent_num_frames = (num_frames - 1) // self.temporal_compression_ratio + 1
+
+ tile_latent_min_num_frames = (
+ self.tile_sample_min_num_frames // self.temporal_compression_ratio
+ )
+ tile_latent_stride_num_frames = (
+ self.tile_sample_stride_num_frames // self.temporal_compression_ratio
+ )
+ blend_num_frames = tile_latent_min_num_frames - tile_latent_stride_num_frames
+
+ row = []
+ # for i in range(0, num_frames, self.tile_sample_stride_num_frames):
+ for i in range(
+ 0,
+ num_frames - self.tile_sample_min_num_frames + 1,
+ self.tile_sample_stride_num_frames,
+ ):
+ tile = x[:, :, i : i + self.tile_sample_min_num_frames + 1, :, :]
+ if self.use_tiling and (
+ height > self.tile_sample_min_height
+ or width > self.tile_sample_min_width
+ ):
+ tile = self.tiled_encode(tile)
+ else:
+ tile = self.encoder(tile).clone()
+ tile = self.quant_conv(tile)
+ if i > 0:
+ tile = tile[:, :, 1:, :, :]
+ row.append(tile)
+
+ result_row = []
+ for i, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_t(row[i - 1], tile, blend_num_frames)
+ t_lim = (
+ tile_latent_min_num_frames
+ if i == len(row) - 1
+ else tile_latent_stride_num_frames
+ )
+ result_row.append(tile[:, :, :t_lim, :, :])
+ else:
+ result_row.append(tile[:, :, : tile_latent_stride_num_frames + 1, :, :])
+
+ enc = torch.cat(result_row, dim=2)[:, :, :latent_num_frames]
+ return enc
+
+ def _temporal_tiled_decode(
+ self, z: torch.Tensor, return_dict: bool = True
+ ) -> Union[DecoderOutput, torch.Tensor]:
+ _, _, num_frames, _, _ = z.shape
+ num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1
+
+ tile_latent_min_height = (
+ self.tile_sample_min_height // self.spatial_compression_ratio
+ )
+ tile_latent_min_width = (
+ self.tile_sample_min_width // self.spatial_compression_ratio
+ )
+ tile_latent_min_num_frames = (
+ self.tile_sample_min_num_frames // self.temporal_compression_ratio
+ )
+ tile_latent_stride_num_frames = (
+ self.tile_sample_stride_num_frames // self.temporal_compression_ratio
+ )
+ blend_num_frames = (
+ self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames
+ )
+
+ row = []
+ for i in range(
+ 0,
+ num_frames - tile_latent_min_num_frames + 1,
+ tile_latent_stride_num_frames,
+ ):
+ tile = z[:, :, i : i + tile_latent_min_num_frames + 1, :, :]
+ if self.use_tiling and (
+ tile.shape[-1] > tile_latent_min_width
+ or tile.shape[-2] > tile_latent_min_height
+ ):
+ decoded = self.tiled_decode(tile, return_dict=True).sample
+ else:
+ tile = self.post_quant_conv(tile)
+ decoded = self.decoder(tile).clone()
+ if i > 0:
+ decoded = decoded[:, :, 1:, :, :]
+ row.append(decoded)
+
+ result_row = []
+ for i, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_t(row[i - 1], tile, blend_num_frames)
+ t_lim = (
+ self.tile_sample_min_num_frames
+ if i == len(row) - 1
+ else self.tile_sample_stride_num_frames
+ )
+ result_row.append(tile[:, :, :t_lim, :, :])
+ else:
+ result_row.append(
+ tile[:, :, : self.tile_sample_stride_num_frames + 1, :, :]
+ )
+
+ dec = torch.cat(result_row, dim=2)[:, :, :num_sample_frames]
+
+ if not return_dict:
+ return (dec,)
+ return DecoderOutput(sample=dec)
+
+ def forward(
+ self,
+ sample: torch.Tensor,
+ sample_posterior: bool = False,
+ return_dict: bool = True,
+ generator: Optional[torch.Generator] = None,
+ ) -> Union[DecoderOutput, torch.Tensor]:
+ r"""
+ Args:
+ sample (`torch.Tensor`): Input sample.
+ sample_posterior (`bool`, *optional*, defaults to `False`):
+ Whether to sample from the posterior.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`DecoderOutput`] instead of a plain tuple.
+ """
+ x = sample
+ posterior = self.encode(x).latent_dist
+ if sample_posterior:
+ z = posterior.sample(generator=generator)
+ else:
+ z = posterior.mode()
+ dec = self.decode(z, return_dict=return_dict)
+ return dec
+
+ def apply_tile_config(self, tile_config):
+ if tile_config is None:
+ return
+ sample_size = tile_config.get("tile_sample_min_size", self.tile_sample_min_height)
+ sample_tsize = tile_config.get("tile_sample_min_tsize", self.tile_sample_min_num_frames)
+ overlap = tile_config.get("tile_overlap_factor", self.tile_overlap_factor)
+ self.tile_overlap_factor = overlap
+
+ stride_size = max(1, int(sample_size * (1 - overlap)))
+ stride_tsize = max(1, int(sample_tsize * (1 - overlap)))
+
+ tile = (1, sample_tsize + 1, sample_size, sample_size)
+ stride = (stride_tsize, stride_size, stride_size)
+ self._forced_tile_config = (tile, stride)
+ self.tile_size = tile
+ self.apply_tiling(tile, stride)
+
+ def apply_tiling(
+ self, tile: Tuple[int, int, int, int], stride: Tuple[int, int, int]
+ ):
+ """Applies tiling."""
+ _, ft, ht, wt = tile
+ fs, hs, ws = stride
+
+ self.use_tiling = True
+ self.tile_sample_min_num_frames = ft - 1
+ self.tile_sample_stride_num_frames = fs
+ self.tile_sample_min_height = ht
+ self.tile_sample_min_width = wt
+ self.tile_sample_stride_height = hs
+ self.tile_sample_stride_width = ws
+
+ def get_VAE_tile_size(self, vae_config, device_mem_capacity, mixed_precision):
+ if mixed_precision:
+ device_mem_capacity /= 1.5
+ if vae_config == 0:
+ if device_mem_capacity >= 24000:
+ use_vae_config = 1
+ elif device_mem_capacity >= 12000:
+ use_vae_config = 2
+ else:
+ use_vae_config = 3
+ else:
+ use_vae_config = vae_config
+
+ if use_vae_config == 1:
+ sample_tsize = 32
+ sample_size = 256
+ elif use_vae_config == 2:
+ sample_tsize = 16
+ sample_size = 256
+ else:
+ sample_tsize = 16
+ sample_size = 192
+
+ return {
+ "tile_sample_min_tsize": sample_tsize,
+ "tile_latent_min_tsize": sample_tsize // self.temporal_compression_ratio,
+ "tile_sample_min_size": sample_size,
+ "tile_latent_min_size": sample_size // self.spatial_compression_ratio,
+ "tile_overlap_factor": 0.25,
+ }
+
+ def get_enc_optimal_tiling(
+ self, shape: List[int]
+ ) -> Tuple[Tuple[int, int, int, int], Tuple[int, int, int]]:
+ """Returns optimal tiling for given shape."""
+ h, w = shape[3:]
+
+ free_mem = torch.cuda.mem_get_info()[0]
+ max_area = free_mem / 256 / 17 / 8
+ num_vals = 256 * 17 * (h + 32) * (w + 32)
+
+ if h * w < max_area and num_vals < 2**31:
+ return (1, 17, h, w), (8, h, w)
+
+ def factorize(n, k):
+ a = sqrt(n / k)
+ b = sqrt(n * k)
+ return ceil(a), ceil(b)
+
+ k = max(h / w, w / h)
+ N = max(ceil(h * w / max_area), ceil(num_vals / 2**31))
+ a, b = factorize(N, k)
+ if h >= w:
+ wn, hn = a, b
+ else:
+ wn, hn = b, a
+
+ if wn > 1:
+ wt = ceil(w / wn / 8) * 8 + 16
+ ws = wt - 32
+ else:
+ wt = w
+ ws = w
+ if hn > 1:
+ ht = ceil(h / hn / 8) * 8 + 16
+ hs = ht - 32
+ else:
+ ht = h
+ hs = h
+ return (1, 17, ht, wt), (8, hs, ws)
+
+ def get_dec_optimal_tiling(
+ self, shape: List[int]
+ ) -> Tuple[Tuple[int, int, int, int], Tuple[int, int, int]]:
+ """Returns optimal tiling for given shape."""
+ b, _, f, h, w = shape
+ enc_inp_shape = [b, 3, 4 * (f - 1) + 1, 8 * h, 8 * w]
+ return self.get_enc_optimal_tiling(enc_inp_shape)
+
+
+def build_vae(conf):
+ if conf.name == "hunyuan":
+ checkpoint_path = getattr(conf, "checkpoint_path", None)
+ if checkpoint_path is None:
+ raise ValueError("Missing VAE checkpoint_path; provide via defaults or code.")
+ config_path = getattr(conf, "config_path", None)
+ if config_path is None and os.path.isfile(checkpoint_path):
+ config_path = os.path.join(os.path.dirname(checkpoint_path), "config.json")
+
+ model_class = AutoencoderKLHunyuanVideo
+ if config_path is not None and os.path.isfile(config_path):
+ try:
+ with open(config_path, "r", encoding="utf-8") as reader:
+ cfg = json.load(reader)
+ if cfg.get("_class_name") == "AutoencoderKLCausal3D" or (
+ "DownEncoderBlockCausal3D" in cfg.get("down_block_types", [])
+ ):
+ from models.hyvideo.vae.autoencoder_kl_causal_3d import AutoencoderKLCausal3D
+ model_class = AutoencoderKLCausal3D
+ except Exception:
+ pass
+
+ if os.path.isfile(checkpoint_path):
+ from mmgp import offload
+ return offload.fast_load_transformers_model(
+ checkpoint_path,
+ writable_tensors=True,
+ modelClass=model_class,
+ defaultConfigPath=config_path,
+ default_dtype=torch.float16,
+ )
+ return AutoencoderKLHunyuanVideo.from_pretrained(
+ checkpoint_path, subfolder="vae", torch_dtype=torch.float16
+ )
+ elif conf.name == "flux":
+ from diffusers.models import AutoencoderKL
+ vae = AutoencoderKL.from_pretrained(conf.checkpoint_path, subfolder="flux/vae", torch_dtype=torch.bfloat16)
+ return vae
+ else:
+ assert False, f"unknown vae name {conf.name}"
diff --git a/models/kandinsky5/kandinsky/pipeline.py b/models/kandinsky5/kandinsky/pipeline.py
new file mode 100644
index 0000000..4885f47
--- /dev/null
+++ b/models/kandinsky5/kandinsky/pipeline.py
@@ -0,0 +1,699 @@
+import json
+import logging
+import struct
+from typing import Optional, Union
+
+import transformers
+import torch
+import torchvision
+import torchvision.transforms.functional as F
+from peft import PeftConfig, LoraConfig, inject_adapter_in_model, set_peft_model_state_dict
+from PIL import Image
+from safetensors.torch import load_file
+from torchvision.transforms import ToPILImage
+
+from .generation_utils import generate_sample, generate_sample_i2v, generate_sample_ti2i
+from .magcache_utils import compute_magcache_threshold, set_magcache_params
+
+torch._dynamo.config.suppress_errors = True
+torch._dynamo.config.verbose = True
+
+
+def read_safetensors_json(file_path):
+ """Reads the metadata (JSON header) from a safetensors file."""
+ with open(file_path, "rb") as f:
+ header_size_bytes = f.read(8)
+ header_size = struct.unpack("Q", header_size_bytes)[0]
+ header_bytes = f.read(header_size)
+ header_str = header_bytes.decode("utf-8")
+ return json.loads(header_str)
+
+
+def get_first_frame_from_image(image, vae, device, max_area, divisibility):
+ if isinstance(image, str):
+ pil_image = Image.open(image).convert("RGB")
+ elif isinstance(image, Image.Image):
+ pil_image = image
+ else:
+ raise ValueError(f"unknown image type: {type(image)}")
+
+ image = F.pil_to_tensor(pil_image).unsqueeze(0)
+ k = 1.0
+ image = image / 127.5 - 1.0
+
+ with torch.no_grad():
+ image = image.to(device=device, dtype=torch.float16).transpose(0, 1).unsqueeze(0)
+ opt_tiling = bool(
+ getattr(vae, "_use_vae_tiling", False)
+ or getattr(vae, "_forced_tile_config", None) is not None
+ )
+ try:
+ enc_out = vae.encode(image, opt_tiling=opt_tiling)
+ except TypeError:
+ if opt_tiling and hasattr(vae, "enable_tiling"):
+ vae.enable_tiling(True)
+ elif not opt_tiling and hasattr(vae, "disable_tiling"):
+ vae.disable_tiling()
+ enc_out = vae.encode(image)
+ lat_image = (
+ enc_out.latent_dist.sample()
+ .squeeze(0)
+ .permute(1, 2, 3, 0)
+ )
+ lat_image = lat_image * vae.config.scaling_factor
+
+ return pil_image, lat_image, k
+
+
+def find_nearest(available_res, real_res):
+ nearest_index = torch.argmin(
+ torch.tensor([abs((x[0] / x[1]) - (real_res[0] / real_res[1])) for x in available_res])
+ ).item()
+ return available_res[nearest_index]
+
+
+class Kandinsky5Pipeline:
+ VIDEO_RESOLUTIONS = {
+ 512: [(512, 512), (512, 768), (768, 512)],
+ 1024: [
+ (1024, 1024),
+ (640, 1408),
+ (1408, 640),
+ (768, 1280),
+ (1280, 768),
+ (896, 1152),
+ (1152, 896),
+ ],
+ }
+ IMAGE_RESOLUTIONS = {
+ 1024: [
+ (1024, 1024),
+ (640, 1408),
+ (1408, 640),
+ (768, 1280),
+ (1280, 768),
+ (896, 1152),
+ (1152, 896),
+ ],
+ }
+ VIDEO_SCALE_FACTORS = {
+ 512: (1.0, 2.0, 2.0),
+ 1024: (1.0, 3.16, 3.16),
+ }
+ IMAGE_SCALE_FACTORS = {
+ 512: (1.0, 2.0, 2.0),
+ 1024: (1.0, 2.0, 2.0),
+ }
+ VIDEO_BUCKET_AREAS = {
+ 512: 512 * 768,
+ 1024: 1024 * 1024,
+ }
+ IMAGE_BUCKET_AREAS = {
+ 512: 512 * 512,
+ 1024: 1024 * 1024,
+ }
+
+ def __init__(
+ self,
+ mode: str,
+ device_map: Union[str, torch.device, dict],
+ dit,
+ text_embedder,
+ vae,
+ resolution: Optional[int] = None,
+ conf=None,
+ ):
+ self.mode = mode
+ self.dit = dit
+ self.text_embedder = text_embedder
+ self.vae = vae
+
+ self.resolution = resolution if resolution is not None else conf.metrics.resolution
+
+ self.device_map = device_map
+ self.conf = conf
+ self.num_steps = conf.model.num_steps
+ self.guidance_weight = conf.model.guidance_weight
+
+ self._interrupt = False
+ if hasattr(self.dit, "_interrupt_check"):
+ self.dit._interrupt_check = lambda: self._interrupt
+
+ if self.mode == "i2v":
+ self.max_area = 512 * 768 if self.resolution == 512 else 1024 * 1024
+ self.divisibility = 16 if self.resolution == 512 else 128
+
+ self._hf_peft_config_loaded = False
+ self.peft_config = {}
+ self.peft_triggers = {}
+ self.peft_trigger = ""
+
+ def _resolve_resolution_bucket(self, width, height, bucket_areas):
+ if width is None or height is None:
+ return None
+ area = width * height
+ return min(bucket_areas, key=lambda res: abs(area - bucket_areas[res]))
+
+ def _apply_runtime_resolution(self, width, height):
+ is_video = self.mode in ("t2v", "i2v")
+ bucket_areas = self.VIDEO_BUCKET_AREAS if is_video else self.IMAGE_BUCKET_AREAS
+ runtime_res = self._resolve_resolution_bucket(width, height, bucket_areas)
+ if runtime_res is None or runtime_res == self.resolution:
+ return
+ self.resolution = runtime_res
+ if hasattr(self.conf, "metrics"):
+ self.conf.metrics.resolution = runtime_res
+ scale_factor = (
+ self.VIDEO_SCALE_FACTORS.get(runtime_res)
+ if is_video
+ else self.IMAGE_SCALE_FACTORS.get(runtime_res)
+ )
+ if scale_factor is not None:
+ self.conf.metrics.scale_factor = list(scale_factor)
+ if self.mode == "i2v":
+ self.max_area = 512 * 768 if runtime_res == 512 else 1024 * 1024
+ self.divisibility = 16 if runtime_res == 512 else 128
+
+ def _disable_magcache(self):
+ if getattr(self.dit, "_magcache_enabled", False):
+ orig_forward = getattr(self.dit, "_magcache_orig_forward", None)
+ if orig_forward is not None:
+ self.dit.forward = orig_forward
+ self.dit._magcache_enabled = False
+
+ def _apply_magcache(self, num_steps, guidance_weight):
+ cache = getattr(self.dit, "cache", None)
+ if cache is None or getattr(cache, "cache_type", None) != "mag":
+ self._disable_magcache()
+ return False
+ mag_ratios = getattr(cache, "def_mag_ratios", None)
+ if not mag_ratios:
+ self._disable_magcache()
+ return False
+ if not hasattr(self.dit, "_magcache_orig_forward"):
+ self.dit._magcache_orig_forward = self.dit.forward
+ no_cfg = abs(guidance_weight - 1.0) <= 1e-6
+ speed_factor = getattr(cache, "multiplier", None)
+ start_step = getattr(cache, "start_step", 0)
+ magcache_K = getattr(cache, "magcache_K", 2)
+ retention_ratio = getattr(cache, "magcache_retention_ratio", 0.2)
+ magcache_thresh = getattr(cache, "magcache_thresh", None)
+ if speed_factor is not None and speed_factor > 0:
+ magcache_thresh = compute_magcache_threshold(
+ mag_ratios,
+ num_steps,
+ speed_factor,
+ start_step=start_step,
+ no_cfg=no_cfg,
+ magcache_K=magcache_K,
+ retention_ratio=retention_ratio,
+ )
+ cache.magcache_thresh = magcache_thresh
+ set_magcache_params(
+ self.dit,
+ mag_ratios,
+ num_steps,
+ no_cfg,
+ start_step=start_step,
+ magcache_thresh=magcache_thresh,
+ magcache_K=magcache_K,
+ retention_ratio=retention_ratio,
+ )
+ self.dit._magcache_enabled = True
+ return True
+
+ def _expand_prompt_t2v(self, prompt):
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": f"""You are a prompt beautifier that transforms short user video descriptions into rich, detailed English prompts specifically optimized for video generation models.
+Here are some example descriptions from the dataset that the model was trained:
+1. "In a dimly lit room with a cluttered background, papers are pinned to the wall and various objects rest on a desk. Three men stand present: one wearing a red sweater, another in a black sweater, and the third in a gray shirt. The man in the gray shirt speaks and makes hand gestures, while the other two men look forward. The camera remains stationary, focusing on the three men throughout the sequence. A gritty and realistic visual style prevails, marked by a greenish tint that contributes to a moody atmosphere. Low lighting casts shadows, enhancing the tense mood of the scene."
+2. "In an office setting, a man sits at a desk wearing a gray sweater and seated in a black office chair. A wooden cabinet with framed pictures stands beside him, alongside a small plant and a lit desk lamp. Engaged in a conversation, he makes various hand gestures to emphasize his points. His hands move in different positions, indicating different ideas or points. The camera remains stationary, focusing on the man throughout. Warm lighting creates a cozy atmosphere. The man appears to be explaining something. The overall visual style is professional and polished, suitable for a business or educational context."
+3. "A person works on a wooden object resembling a sunburst pattern, holding it in their left hand while using their right hand to insert a thin wire into the gaps between the wooden pieces. The background features a natural outdoor setting with greenery and a tree trunk visible. The camera stays focused on the hands and the wooden object throughout, capturing the detailed process of assembling the wooden structure. The person carefully threads the wire through the gaps, ensuring the wooden pieces are securely fastened together. The scene unfolds with a naturalistic and instructional style, emphasizing the craftsmanship and the methodical steps taken to complete the task."
+IImportantly! These are just examples from a large training dataset of 200 million videos.
+Rewrite Prompt: "{prompt}" to get high-quality video generation. Answer only with expanded prompt.""",
+ },
+ ],
+ }
+ ]
+ text = self.text_embedder.embedder.processor.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True
+ )
+ inputs = self.text_embedder.embedder.processor(
+ text=[text],
+ images=None,
+ videos=None,
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to(self.text_embedder.embedder.model.device)
+ generated_ids = self.text_embedder.embedder.model.generate(
+ **inputs, max_new_tokens=256
+ )
+ generated_ids_trimmed = [
+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ ]
+ output_text = self.text_embedder.embedder.processor.batch_decode(
+ generated_ids_trimmed,
+ skip_special_tokens=True,
+ clean_up_tokenization_spaces=False,
+ )
+ return output_text[0]
+
+ def _expand_prompt_t2i(self, prompt):
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": f"""Rewrite and enhance the original prompt with richer detail, clearer structure, and improved descriptive quality. Expand the scene, atmosphere, and context while preserving the user's intent. When adding text that should appear inside an image, place that text inside double quotes and in capital letters. Strengthen visual clarity, style, and specificity, but do not change the meaning. Output only the enhanced prompt, written in polished, vivid language suitable for high-quality image generation.
+example:
+Original text: white mini police car with blue stripes, with 911 and 'Police' text
+Result: A miniature model car simulating the official transport of the relevant authorities. The body is white with blue stripes. The word "911" is written in large blue letters on the hood and side. Below it, "POLICE" is used in a font. The windows are transparent, and the interior has black seats. The headlights have plastic lenses, and the roof has blue and red beacons. The radiator grille has vertical slots. The wheels are black with white rims. The doors are closed, the windows have black frames. The background is uniform white.
+Here 911 in double quotes because it is text on image, 'Police' -> "POLICE" because it should be in double quotes and capital letters.
+Rewrite Prompt: "{prompt}". Answer only with expanded prompt.""",
+ },
+ ],
+ }
+ ]
+ text = self.text_embedder.embedder.processor.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True
+ )
+ inputs = self.text_embedder.embedder.processor(
+ text=[text],
+ images=None,
+ videos=None,
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to(self.text_embedder.embedder.model.device)
+ generated_ids = self.text_embedder.embedder.model.generate(
+ **inputs, max_new_tokens=256
+ )
+ generated_ids_trimmed = [
+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ ]
+ output_text = self.text_embedder.embedder.processor.batch_decode(
+ generated_ids_trimmed,
+ skip_special_tokens=True,
+ clean_up_tokenization_spaces=False,
+ )
+ return output_text[0]
+
+ def _expand_prompt_i2i(self, prompt, image):
+ width, height = image.size
+ image = image.resize((width // 4, height // 4))
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": f"""Rewrite and enhance the original editing instruction with richer detail, clearer structure, and improved descriptive quality. When adding text that should appear inside an image, place that text inside double quotes and in capital letters. Explain what needs to be changed and what needs to be left unchanged. Explain in details how to change camera position or tell that camera position shouldn't be changed.
+example:
+Original text: add text 911 and 'Police'
+Result: Add the word "911" in large blue letters to the hood. Below that, add the word "POLICE." Keep the camera position unchanged, as do the background, car position, and lighting.
+Rewrite Prompt: "{prompt}". Answer only with expanded prompt.""",
+ },
+ {
+ "type": "image",
+ "image": image,
+ },
+ ],
+ }
+ ]
+ text = self.text_embedder.embedder.processor.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True
+ )
+ inputs = self.text_embedder.embedder.processor(
+ text=[text],
+ images=image,
+ videos=None,
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to(self.text_embedder.embedder.model.device)
+ generated_ids = self.text_embedder.embedder.model.generate(
+ **inputs, max_new_tokens=256
+ )
+ generated_ids_trimmed = [
+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ ]
+ output_text = self.text_embedder.embedder.processor.batch_decode(
+ generated_ids_trimmed,
+ skip_special_tokens=True,
+ clean_up_tokenization_spaces=False,
+ )
+ return output_text[0]
+
+ def _prepare_seed(self, seed):
+ if seed is not None:
+ return seed
+ seed = torch.randint(2**32 - 1, (1,), device="cpu")
+ return int(seed.item())
+
+ def _maybe_expand_prompt(self, caption, seed, image=None):
+ if not caption:
+ return caption
+ transformers.set_seed(seed)
+ if self.mode == "t2v":
+ caption = self.peft_trigger + self._expand_prompt_t2v(caption)
+ elif self.mode == "i2v":
+ caption = self.text_embedder.embedder.expand_text_prompt(
+ caption, image, device=self.device_map["text_embedder"]
+ )
+ caption = self.peft_trigger + caption
+ elif self.mode == "t2i":
+ caption = self._expand_prompt_t2i(caption)
+ elif self.mode == "i2i":
+ caption = self._expand_prompt_i2i(caption, image=image)
+ return caption
+
+ def __call__(
+ self,
+ text: str,
+ image: Optional[Union[str, Image.Image]] = None,
+ time_length: int = 5,
+ width: Optional[int] = None,
+ height: Optional[int] = None,
+ frame_num: Optional[int] = None,
+ seed: int = None,
+ num_steps: int = None,
+ guidance_weight: float = None,
+ scheduler_scale: Optional[float] = None,
+ negative_caption: str = "",
+ expand_prompts: bool = True,
+ save_path: str = None,
+ progress: bool = True,
+ callback=None,
+ joint_pass: bool = False,
+ ):
+ num_steps = self.num_steps if num_steps is None else num_steps
+ guidance_weight = self.guidance_weight if guidance_weight is None else guidance_weight
+ if scheduler_scale is None:
+ scheduler_scale = 10.0 if self.mode in ("t2v", "i2v") else 3.0
+
+ seed = self._prepare_seed(seed)
+ self._apply_runtime_resolution(width, height)
+ magcache_enabled = self._apply_magcache(num_steps, guidance_weight)
+
+ try:
+ if self.mode in ("t2v", "i2v"):
+ temporal_ratio = getattr(self.vae, "time_compression_ratio", None)
+ if temporal_ratio is None:
+ temporal_ratio = getattr(getattr(self.vae, "config", None), "time_compression_ratio", None)
+ if temporal_ratio is None:
+ temporal_ratio = 4
+ if frame_num is None:
+ num_frames = 1 if time_length == 0 else time_length * 24 // temporal_ratio + 1
+ else:
+ if frame_num <= 0:
+ raise ValueError("frame_num must be a positive integer")
+ if (frame_num - 1) % temporal_ratio != 0:
+ raise ValueError(
+ f"frame_num-1 must be a multiple of {temporal_ratio}, got {frame_num}"
+ )
+ # Convert output frame count to latent frame count.
+ num_frames = (frame_num - 1) // temporal_ratio + 1
+
+ if self.mode == "t2v":
+ if width is None or height is None:
+ raise ValueError("width and height must be provided for t2v generation")
+ caption = text
+ if expand_prompts:
+ caption = self._maybe_expand_prompt(caption, seed)
+ shape = (1, num_frames, height // 8, width // 8, 16)
+ images = generate_sample(
+ shape,
+ caption,
+ self.dit,
+ self.vae,
+ self.conf,
+ text_embedder=self.text_embedder,
+ num_steps=num_steps,
+ guidance_weight=guidance_weight,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ seed=seed,
+ device=self.device_map["dit"],
+ vae_device=self.device_map["vae"],
+ text_embedder_device=self.device_map["text_embedder"],
+ progress=progress,
+ callback=callback,
+ joint_pass=joint_pass,
+ interrupt_check=lambda: self._interrupt,
+ )
+ if images is None:
+ return None
+ elif self.mode == "i2v":
+ if image is None:
+ raise ValueError("image is required for i2v generation")
+ pil_image, image_lat, k = get_first_frame_from_image(
+ image, self.vae, self.device_map["vae"], self.max_area, self.divisibility
+ )
+
+ caption = text
+ if expand_prompts:
+ caption = self._maybe_expand_prompt(caption, seed, image=pil_image)
+
+ height_lat, width_lat = image_lat.shape[1:3]
+ shape = (1, num_frames, height_lat, width_lat, 16)
+ images = generate_sample_i2v(
+ shape,
+ caption,
+ self.dit,
+ self.vae,
+ self.conf,
+ text_embedder=self.text_embedder,
+ images=image_lat,
+ num_steps=num_steps,
+ guidance_weight=guidance_weight,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ seed=seed,
+ device=self.device_map["dit"],
+ vae_device=self.device_map["vae"],
+ progress=progress,
+ callback=callback,
+ joint_pass=joint_pass,
+ interrupt_check=lambda: self._interrupt,
+ )
+ if images is None:
+ return None
+ if k > 16:
+ h, w = images.shape[-2:]
+ images = F.resize(images[0], (int(h / k / 16), int(w / k / 16)))
+ else:
+ if width is None or height is None:
+ if image is None:
+ raise ValueError("width/height or image is required for image generation")
+ if isinstance(image, str):
+ image_obj = Image.open(image)
+ else:
+ image_obj = image
+ height, width = find_nearest(
+ self.IMAGE_RESOLUTIONS[self.resolution], image_obj.size[::-1]
+ )
+
+ caption = text
+ image_tensor = None
+ if image is not None:
+ if isinstance(image, str):
+ image_obj = Image.open(image)
+ elif isinstance(image, Image.Image):
+ image_obj = image
+ else:
+ raise ValueError(f"unknown image type: {type(image)}")
+ image_tensor = F.pil_to_tensor(image_obj)[None]
+ if expand_prompts:
+ caption = self._maybe_expand_prompt(caption, seed, image=image_obj if image is not None else None)
+
+ shape = (1, 1, height // 8, width // 8, 16)
+ images = generate_sample_ti2i(
+ shape,
+ caption,
+ self.dit,
+ self.vae,
+ self.conf,
+ text_embedder=self.text_embedder,
+ num_steps=num_steps,
+ guidance_weight=guidance_weight,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ seed=seed,
+ device=self.device_map["dit"],
+ vae_device=self.device_map["vae"],
+ text_embedder_device=self.device_map["text_embedder"],
+ progress=progress,
+ image_vae=False,
+ image=image_tensor,
+ callback=callback,
+ joint_pass=joint_pass,
+ interrupt_check=lambda: self._interrupt,
+ )
+ if images is None:
+ return None
+ finally:
+ if magcache_enabled:
+ self._disable_magcache()
+
+
+ if self.mode in ("t2v", "i2v") and time_length != 0:
+ if save_path is not None:
+ if isinstance(save_path, str):
+ save_path = [save_path]
+ if len(save_path) == len(images):
+ for path, video in zip(save_path, images):
+ torchvision.io.write_video(
+ path,
+ video.float().permute(1, 2, 3, 0).cpu().numpy(),
+ fps=24,
+ options={"crf": "5"},
+ )
+ return images
+
+ return_images = []
+ for image in images.squeeze(2).cpu():
+ return_images.append(ToPILImage()(image))
+ if save_path is not None:
+ if isinstance(save_path, str):
+ save_path = [save_path]
+ if len(save_path) == len(return_images):
+ for path, out_image in zip(save_path, return_images):
+ out_image.save(path)
+ return return_images
+
+ def load_adapter(
+ self,
+ adapter_config: Union[PeftConfig, str],
+ adapter_path: Optional[str] = None,
+ adapter_name: Optional[str] = None,
+ trigger: Optional[str] = None,
+ ) -> None:
+ if adapter_name is None:
+ adapter_name = "default"
+ if self._hf_peft_config_loaded and adapter_name in self.peft_config:
+ raise ValueError(f"Adapter with name {adapter_name} already exists. Please use a different name.")
+
+ if not isinstance(adapter_config, PeftConfig):
+ try:
+ with open(adapter_config, "r") as f:
+ adapter_config = json.load(f)
+ adapter_config = LoraConfig(**adapter_config)
+ except Exception as exc:
+ raise TypeError(
+ "adapter_config should be an instance of PeftConfig or a path to a json file."
+ ) from exc
+ self.peft_config[adapter_name] = adapter_config
+
+ inject_adapter_in_model(adapter_config, self.dit, adapter_name)
+
+ if not self._hf_peft_config_loaded:
+ self._hf_peft_config_loaded = True
+ adapter_state_dict = load_file(adapter_path)
+ adapter_metadata = read_safetensors_json(adapter_path)
+ if trigger is not None:
+ self.peft_trigger = trigger
+ else:
+ if "__metadata__" in adapter_metadata and "trigger" in adapter_metadata["__metadata__"]:
+ self.peft_trigger = adapter_metadata["__metadata__"]["trigger"]
+ else:
+ self.peft_trigger = ""
+ self.peft_triggers[adapter_name] = self.peft_trigger
+
+ processed_adapter_state_dict = {}
+ for key, value in adapter_state_dict.items():
+ new_key = key
+ for prefix in ["base_model.model.", "transformer."]:
+ if new_key.startswith(prefix):
+ new_key = new_key[len(prefix) :]
+ break
+
+ new_key = new_key.replace(".default", "")
+ processed_adapter_state_dict[new_key] = value
+
+ incompatible_keys = set_peft_model_state_dict(
+ self.dit, processed_adapter_state_dict, adapter_name
+ )
+
+ if incompatible_keys is not None:
+ err_msg = ""
+ if hasattr(incompatible_keys, "unexpected_keys") and len(incompatible_keys.unexpected_keys) > 0:
+ err_msg = (
+ f"Loading adapter weights from state_dict led to unexpected keys not found in the model: "
+ f"{', '.join(incompatible_keys.unexpected_keys)}. "
+ )
+
+ missing_keys = getattr(incompatible_keys, "missing_keys", None)
+ if missing_keys:
+ lora_missing_keys = [k for k in missing_keys if "lora_" in k and adapter_name in k]
+ if lora_missing_keys:
+ err_msg += (
+ f"Loading adapter weights from state_dict led to missing keys in the model: "
+ f"{', '.join(lora_missing_keys)}"
+ )
+
+ if err_msg:
+ logging.warning(err_msg)
+
+ self.set_adapter(adapter_name)
+
+ def set_adapter(self, adapter_name: Union[list[str], str]) -> None:
+ if not self._hf_peft_config_loaded:
+ raise ValueError("No adapter loaded. Please load an adapter first.")
+ elif isinstance(adapter_name, list):
+ missing = set(adapter_name) - set(self.peft_config)
+ if len(missing) > 0:
+ raise ValueError(
+ "Following adapter(s) could not be found: "
+ f"{', '.join(missing)}. "
+ f"current loaded adapters are: {list(self.peft_config.keys())}"
+ )
+ elif adapter_name not in self.peft_config:
+ raise ValueError(
+ f"Adapter with name {adapter_name} not found. Please pass the correct adapter name among {list(self.peft_config.keys())}"
+ )
+
+ from peft.tuners.tuners_utils import BaseTunerLayer
+ from peft.utils import ModulesToSaveWrapper
+
+ adapters_set = False
+
+ for _, module in self.dit.named_modules():
+ if isinstance(module, BaseTunerLayer):
+ if hasattr(module, "enable_adapters"):
+ module.enable_adapters(enabled=True)
+ else:
+ module.disable_adapters = False
+ if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):
+ if hasattr(module, "set_adapter"):
+ module.set_adapter(adapter_name)
+ else:
+ module.active_adapter = adapter_name
+ adapters_set = True
+
+ if not adapters_set:
+ raise ValueError(
+ "Did not succeed in setting the adapter. Please make sure you are using a model that supports adapters."
+ )
+ self.peft_trigger = self.peft_triggers[adapter_name]
+
+ def disable_adapters(self) -> None:
+ if not self._hf_peft_config_loaded:
+ raise ValueError("No adapter loaded. Please load an adapter first.")
+
+ from peft.tuners.tuners_utils import BaseTunerLayer
+ from peft.utils import ModulesToSaveWrapper
+
+ for _, module in self.dit.named_modules():
+ if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):
+ if hasattr(module, "enable_adapters"):
+ module.enable_adapters(enabled=False)
+ else:
+ module.disable_adapters = True
+ self.peft_trigger = ""
diff --git a/models/kandinsky5/kandinsky/t2i_pipeline.py b/models/kandinsky5/kandinsky/t2i_pipeline.py
new file mode 100644
index 0000000..6068bf1
--- /dev/null
+++ b/models/kandinsky5/kandinsky/t2i_pipeline.py
@@ -0,0 +1,61 @@
+from typing import Optional, Union
+
+import torch
+
+from .pipeline import Kandinsky5Pipeline
+
+
+class Kandinsky5T2IPipeline(Kandinsky5Pipeline):
+ RESOLUTIONS = Kandinsky5Pipeline.IMAGE_RESOLUTIONS
+
+ def __init__(
+ self,
+ device_map: Union[str, torch.device, dict],
+ dit,
+ text_embedder,
+ vae,
+ resolution: int = 1024,
+ conf=None,
+ ):
+ super().__init__(
+ mode="t2i",
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ resolution=resolution,
+ conf=conf,
+ )
+
+ def __call__(
+ self,
+ text: str,
+ width: int = 1024,
+ height: int = 1024,
+ seed: int = None,
+ num_steps: Optional[int] = None,
+ guidance_weight: Optional[float] = None,
+ scheduler_scale: float = 3.0,
+ negative_caption: str = "",
+ expand_prompts: bool = True,
+ save_path: str = None,
+ progress: bool = True,
+ joint_pass: bool = False,
+ ):
+ return super().__call__(
+ text=text,
+ image=None,
+ time_length=0,
+ width=width,
+ height=height,
+ frame_num=1,
+ seed=seed,
+ num_steps=num_steps,
+ guidance_weight=guidance_weight,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ expand_prompts=expand_prompts,
+ save_path=save_path,
+ progress=progress,
+ joint_pass=joint_pass,
+ )
diff --git a/models/kandinsky5/kandinsky/t2v_pipeline.py b/models/kandinsky5/kandinsky/t2v_pipeline.py
new file mode 100644
index 0000000..d7bcd94
--- /dev/null
+++ b/models/kandinsky5/kandinsky/t2v_pipeline.py
@@ -0,0 +1,65 @@
+from typing import Optional, Union
+
+import torch
+
+from .pipeline import Kandinsky5Pipeline
+
+
+class Kandinsky5T2VPipeline(Kandinsky5Pipeline):
+ RESOLUTIONS = Kandinsky5Pipeline.VIDEO_RESOLUTIONS
+
+ def __init__(
+ self,
+ device_map: Union[str, torch.device, dict],
+ dit,
+ text_embedder,
+ vae,
+ conf=None,
+ ):
+ super().__init__(
+ mode="t2v",
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ resolution=conf.metrics.resolution,
+ conf=conf,
+ )
+
+ def __call__(
+ self,
+ text: str,
+ time_length: int = 5,
+ width: int = 768,
+ height: int = 512,
+ frame_num: Optional[int] = None,
+ seed: int = None,
+ num_steps: int = None,
+ guidance_weight: float = None,
+ scheduler_scale: float = 10.0,
+ negative_caption: str = (
+ "Static, 2D cartoon, cartoon, 2d animation, paintings, images, worst quality, "
+ "low quality, ugly, deformed, walking backwards"
+ ),
+ expand_prompts: bool = True,
+ save_path: str = None,
+ progress: bool = True,
+ joint_pass: bool = False,
+ ):
+ return super().__call__(
+ text=text,
+ image=None,
+ time_length=time_length,
+ width=width,
+ height=height,
+ frame_num=frame_num,
+ seed=seed,
+ num_steps=num_steps,
+ guidance_weight=guidance_weight,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ expand_prompts=expand_prompts,
+ save_path=save_path,
+ progress=progress,
+ joint_pass=joint_pass,
+ )
diff --git a/models/kandinsky5/kandinsky/utils.py b/models/kandinsky5/kandinsky/utils.py
new file mode 100644
index 0000000..c870eb7
--- /dev/null
+++ b/models/kandinsky5/kandinsky/utils.py
@@ -0,0 +1,503 @@
+import os
+from typing import Optional, Union
+
+import numpy as np
+import torch
+
+from huggingface_hub import hf_hub_download, snapshot_download
+from omegaconf import OmegaConf
+from omegaconf.dictconfig import DictConfig
+
+from .models.dit import get_dit
+from .models.text_embedders import get_text_embedder
+from .models.vae import build_vae
+from .i2v_pipeline import Kandinsky5I2VPipeline
+from .t2v_pipeline import Kandinsky5T2VPipeline
+from .t2i_pipeline import Kandinsky5T2IPipeline
+from .i2i_pipeline import Kandinsky5I2IPipeline
+from .magcache_utils import set_magcache_params
+
+from PIL import Image
+from safetensors.torch import load_file
+
+torch._dynamo.config.suppress_errors = True
+
+
+HF_TOKEN = None
+
+
+def get_hf_token():
+ return HF_TOKEN
+
+
+def set_hf_token(hf_token):
+ global HF_TOKEN
+ HF_TOKEN = hf_token
+
+
+def get_T2V_pipeline(
+ device_map: Union[str, torch.device, dict],
+ cache_dir: str = "./weights/",
+ dit_path: str = None,
+ text_encoder_path: str = None,
+ text_encoder2_path: str = None,
+ vae_path: str = None,
+ conf_path: str = None,
+ magcache: bool = False,
+ quantized_qwen: bool = False,
+ text_token_padding: bool = False,
+ attention_engine: str = "auto",
+) -> Kandinsky5T2VPipeline:
+ if not isinstance(device_map, dict):
+ device_map = {"dit": device_map, "vae": device_map, "text_embedder": device_map}
+
+ os.makedirs(cache_dir, exist_ok=True)
+
+ if dit_path is None and conf_path is None:
+ dit_path = snapshot_download(
+ repo_id="kandinskylab/Kandinsky-5.0-T2V-Lite-sft-5s",
+ allow_patterns="model/*",
+ local_dir=cache_dir,
+ token=get_hf_token()
+ )
+ dit_path = os.path.join(cache_dir, "model/kandinsky5lite_t2v_sft_5s.safetensors")
+
+ if vae_path is None and conf_path is None:
+ vae_path = snapshot_download(
+ repo_id="hunyuanvideo-community/HunyuanVideo",
+ allow_patterns="vae/*",
+ local_dir=cache_dir,
+ token=get_hf_token()
+ )
+ vae_path = os.path.join(cache_dir, "vae/")
+
+ if text_encoder_path is None and conf_path is None:
+ text_encoder_path = snapshot_download(
+ repo_id="Qwen/Qwen2.5-VL-7B-Instruct",
+ local_dir=os.path.join(cache_dir, "text_encoder/"),
+ token=get_hf_token()
+ )
+ text_encoder_path = os.path.join(cache_dir, "text_encoder/")
+
+ if text_encoder2_path is None and conf_path is None:
+ text_encoder2_path = snapshot_download(
+ repo_id="openai/clip-vit-large-patch14",
+ local_dir=os.path.join(cache_dir, "text_encoder2/"),
+ token=get_hf_token()
+ )
+ text_encoder2_path = os.path.join(cache_dir, "text_encoder2/")
+
+ if conf_path is None:
+ conf = get_default_conf(
+ dit_path, vae_path, text_encoder_path, text_encoder2_path
+ )
+ else:
+ conf = OmegaConf.load(conf_path)
+ conf.model.dit_params.attention_engine = attention_engine
+
+ conf.model.text_embedder.qwen.mode = "t2v"
+ text_embedder = get_text_embedder(conf.model.text_embedder, device=torch.device("cpu"),
+ quantized_qwen=quantized_qwen, text_token_padding=text_token_padding)
+
+ vae = build_vae(conf.model.vae)
+ vae = vae.eval()
+
+ dit = get_dit(conf.model.dit_params, text_token_padding=text_token_padding)
+
+ if magcache:
+ mag_ratios = conf.magcache.mag_ratios
+ num_steps = conf.model.num_steps
+ no_cfg = False
+ if conf.model.guidance_weight == 1.0:
+ no_cfg = True
+ set_magcache_params(dit, mag_ratios, num_steps, no_cfg, start_step=0)
+
+ state_dict = load_file(conf.model.checkpoint_path, device='cpu')
+ dit.load_state_dict(state_dict, assign=True)
+
+ return Kandinsky5T2VPipeline(
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ conf=conf,
+ )
+
+def get_I2V_pipeline(
+ device_map: Union[str, torch.device, dict],
+ cache_dir: str = "./weights/",
+ dit_path: str = None,
+ text_encoder_path: str = None,
+ text_encoder2_path: str = None,
+ vae_path: str = None,
+ conf_path: str = None,
+ magcache: bool = False,
+ quantized_qwen: bool = False,
+ text_token_padding: bool = False,
+ attention_engine: str = "auto",
+) -> Kandinsky5T2VPipeline:
+ if not isinstance(device_map, dict):
+ device_map = {"dit": device_map, "vae": device_map, "text_embedder": device_map}
+
+ os.makedirs(cache_dir, exist_ok=True)
+
+ if dit_path is None and conf_path is None:
+ dit_path = snapshot_download(
+ repo_id="kandinskylab/Kandinsky-5.0-I2V-Lite-5s",
+ allow_patterns="model/*",
+ local_dir=cache_dir,
+ token=get_hf_token()
+ )
+ dit_path = os.path.join(cache_dir, "model/kandinsky5lite_i2v_5s.safetensors")
+
+ if vae_path is None and conf_path is None:
+ vae_path = snapshot_download(
+ repo_id="hunyuanvideo-community/HunyuanVideo",
+ allow_patterns="vae/*",
+ local_dir=cache_dir,
+ token=get_hf_token()
+ )
+ vae_path = os.path.join(cache_dir, "vae/")
+
+ if text_encoder_path is None and conf_path is None:
+ text_encoder_path = snapshot_download(
+ repo_id="Qwen/Qwen2.5-VL-7B-Instruct",
+ local_dir=os.path.join(cache_dir, "text_encoder/"),
+ token=get_hf_token()
+ )
+ text_encoder_path = os.path.join(cache_dir, "text_encoder/")
+
+ if text_encoder2_path is None and conf_path is None:
+ text_encoder2_path = snapshot_download(
+ repo_id="openai/clip-vit-large-patch14",
+ local_dir=os.path.join(cache_dir, "text_encoder2/"),
+ token=get_hf_token()
+ )
+ text_encoder2_path = os.path.join(cache_dir, "text_encoder2/")
+
+ if conf_path is None:
+ conf = get_default_conf(
+ dit_path, vae_path, text_encoder_path, text_encoder2_path
+ )
+ else:
+ conf = OmegaConf.load(conf_path)
+ conf.model.dit_params.attention_engine = attention_engine
+
+ conf.model.text_embedder.qwen.mode = "i2v"
+ text_embedder = get_text_embedder(conf.model.text_embedder, device=torch.device("cpu"),
+ quantized_qwen=quantized_qwen, text_token_padding=text_token_padding)
+
+ vae = build_vae(conf.model.vae)
+ vae = vae.eval()
+
+ dit = get_dit(conf.model.dit_params, text_token_padding=text_token_padding)
+
+ if magcache:
+ mag_ratios = conf.magcache.mag_ratios
+ num_steps = conf.model.num_steps
+ no_cfg = False
+ if conf.model.guidance_weight == 1.0:
+ no_cfg = True
+ set_magcache_params(dit, mag_ratios, num_steps, no_cfg, start_step=0)
+
+ state_dict = load_file(conf.model.checkpoint_path, device='cpu')
+ dit.load_state_dict(state_dict, assign=True)
+
+ return Kandinsky5I2VPipeline(
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ conf=conf,
+ )
+
+
+def _get_TI2I_params(
+ instruct_type: bool,
+ model_name: str,
+ weights_name: str,
+ device_map: Union[str, torch.device, dict],
+ resolution: int = 1024,
+ cache_dir: str = "./weights/",
+ dit_path: str = None,
+ text_encoder_path: str = None,
+ text_encoder2_path: str = None,
+ vae_path: str = None,
+ conf_path: str = None,
+ magcache: bool = False,
+ quantized_qwen: bool = False,
+ text_token_padding: bool = False,
+ attention_engine: str = "auto",
+) -> Kandinsky5T2IPipeline:
+ assert resolution in [1024]
+
+ if not isinstance(device_map, dict):
+ device_map = {"dit": device_map, "vae": device_map, "text_embedder": device_map}
+
+ os.makedirs(cache_dir, exist_ok=True)
+
+ if dit_path is None and conf_path is None:
+ dit_path = snapshot_download(
+ repo_id=f"kandinskylab/{model_name}",
+ allow_patterns="model/*",
+ local_dir=cache_dir,
+ token=get_hf_token()
+ )
+ dit_path = os.path.join(cache_dir, f"model/{weights_name}")
+
+ if vae_path is None and conf_path is None:
+ vae_path = snapshot_download(
+ repo_id="hunyuanvideo-community/HunyuanVideo",
+ allow_patterns="vae/*",
+ local_dir=cache_dir,
+ token=get_hf_token()
+ )
+ vae_path = os.path.join(cache_dir, "vae/")
+
+ if text_encoder_path is None and conf_path is None:
+ text_encoder_path = snapshot_download(
+ repo_id="Qwen/Qwen2.5-VL-7B-Instruct",
+ local_dir=os.path.join(cache_dir, "text_encoder/"),
+ token=get_hf_token()
+ )
+ text_encoder_path = os.path.join(cache_dir, "text_encoder/")
+
+ if text_encoder2_path is None and conf_path is None:
+ text_encoder2_path = snapshot_download(
+ repo_id="openai/clip-vit-large-patch14",
+ local_dir=os.path.join(cache_dir, "text_encoder2/"),
+ token=get_hf_token()
+ )
+ text_encoder2_path = os.path.join(cache_dir, "text_encoder2/")
+
+ if conf_path is None:
+ conf = get_default_ti2i_conf(
+ dit_path, vae_path, text_encoder_path, text_encoder2_path, instruct_type=instruct_type,
+ )
+ else:
+ conf = OmegaConf.load(conf_path)
+ conf.model.dit_params.attention_engine = attention_engine
+
+ conf.model.text_embedder.qwen.mode = "t2i"
+ text_embedder = get_text_embedder(conf.model.text_embedder, device=torch.device("cpu"),
+ quantized_qwen=quantized_qwen, text_token_padding=text_token_padding)
+
+ vae = build_vae(conf.model.vae)
+ vae = vae.eval()
+
+ dit = get_dit(conf.model.dit_params, text_token_padding=text_token_padding)
+
+ if magcache:
+ mag_ratios = conf.magcache.mag_ratios
+ num_steps = conf.model.num_steps
+ no_cfg = False
+ if conf.model.guidance_weight == 1.0:
+ no_cfg = True
+ set_magcache_params(dit, mag_ratios, num_steps, no_cfg, start_step=0)
+
+ state_dict = load_file(conf.model.checkpoint_path)
+ dit.load_state_dict(state_dict, assign=True)
+
+ return dict(
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ resolution=resolution,
+ conf=conf,
+ )
+
+
+def get_T2I_pipeline(
+ device_map: Union[str, torch.device, dict],
+ resolution: int = 1024,
+ cache_dir: str = "./weights/",
+ dit_path: str = None,
+ text_encoder_path: str = None,
+ text_encoder2_path: str = None,
+ vae_path: str = None,
+ conf_path: str = None,
+ magcache: bool = False,
+ quantized_qwen: bool = False,
+ text_token_padding: bool = False,
+ attention_engine: str = "auto",
+) -> Kandinsky5T2IPipeline:
+ kwargs = _get_TI2I_params(
+ instruct_type=None,
+ model_name='Kandinsky-5.0-T2I-Lite',
+ weights_name='kandinsky5lite_t2i.safetensors',
+ device_map=device_map,
+ resolution=resolution,
+ cache_dir=cache_dir,
+ dit_path=dit_path,
+ text_encoder_path=text_encoder_path,
+ text_encoder2_path=text_encoder2_path,
+ vae_path=vae_path,
+ conf_path=conf_path,
+ magcache=magcache,
+ quantized_qwen=quantized_qwen,
+ text_token_padding=text_token_padding,
+ attention_engine=attention_engine,
+ )
+
+ return Kandinsky5T2IPipeline(**kwargs)
+
+
+def get_I2I_pipeline(
+ device_map: Union[str, torch.device, dict],
+ resolution: int = 1024,
+ cache_dir: str = "./weights/",
+ dit_path: str = None,
+ text_encoder_path: str = None,
+ text_encoder2_path: str = None,
+ vae_path: str = None,
+ conf_path: str = None,
+ magcache: bool = False,
+ quantized_qwen: bool = False,
+ text_token_padding: bool = False,
+ attention_engine: str = "auto",
+) -> Kandinsky5T2IPipeline:
+ kwargs = _get_TI2I_params(
+ instruct_type='channel',
+ model_name='Kandinsky-5.0-I2I-Lite',
+ weights_name='kandinsky5lite_i2i.safetensors',
+ device_map=device_map,
+ resolution=resolution,
+ cache_dir=cache_dir,
+ dit_path=dit_path,
+ text_encoder_path=text_encoder_path,
+ text_encoder2_path=text_encoder2_path,
+ vae_path=vae_path,
+ conf_path=conf_path,
+ magcache=magcache,
+ quantized_qwen=quantized_qwen,
+ text_token_padding=text_token_padding,
+ attention_engine=attention_engine,
+ )
+
+ return Kandinsky5I2IPipeline(**kwargs)
+
+
+def get_default_conf(
+ dit_path,
+ vae_path,
+ text_encoder_path,
+ text_encoder2_path,
+) -> DictConfig:
+ dit_params = {
+ "in_visual_dim": 16,
+ "out_visual_dim": 16,
+ "time_dim": 512,
+ "patch_size": [1, 2, 2],
+ "model_dim": 1792,
+ "ff_dim": 7168,
+ "num_text_blocks": 2,
+ "num_visual_blocks": 32,
+ "axes_dims": [16, 24, 24],
+ "visual_cond": True,
+ "in_text_dim": 3584,
+ "in_text_dim2": 768,
+ }
+
+ attention = {
+ "type": "flash",
+ "causal": False,
+ "local": False,
+ "glob": False,
+ "window": 3,
+ }
+
+ vae = {
+ "checkpoint_path": vae_path,
+ "name": "hunyuan",
+ }
+
+ text_embedder = {
+ "qwen": {
+ "emb_size": 3584,
+ "checkpoint_path": text_encoder_path,
+ "max_length": 256,
+ },
+ "clip": {
+ "checkpoint_path": text_encoder2_path,
+ "emb_size": 768,
+ "max_length": 77,
+ },
+ }
+
+ conf = {
+ "model": {
+ "checkpoint_path": dit_path,
+ "vae": vae,
+ "text_embedder": text_embedder,
+ "dit_params": dit_params,
+ "attention": attention,
+ "num_steps": 50,
+ "guidance_weight": 5.0,
+ },
+ "metrics": {"scale_factor": (1, 2, 2), "resolution": 512,},
+ }
+
+ return DictConfig(conf)
+
+
+def get_default_ti2i_conf(
+ dit_path,
+ vae_path,
+ text_encoder_path,
+ text_encoder2_path,
+ instruct_type=None,
+) -> DictConfig:
+ dit_params = {
+ "instruct_type": instruct_type,
+ "in_visual_dim": 16,
+ "out_visual_dim": 16,
+ "time_dim": 512,
+ "patch_size": [1, 2, 2],
+ "model_dim": 2560,
+ "ff_dim": 10240,
+ "num_text_blocks": 2,
+ "num_visual_blocks": 50,
+ "axes_dims": [32,48, 48],
+ }
+
+ attention = {
+ "type": "flash",
+ "causal": False,
+ "local": False,
+ "glob": False,
+ "window": 3,
+ }
+
+ vae = {
+ "checkpoint_path": vae_path,
+ "name": "hunyuan",
+ }
+
+ text_embedder = {
+ "qwen": {
+ "emb_size": 3584,
+ "checkpoint_path": text_encoder_path,
+ "max_length": 512,
+ },
+ "clip": {
+ "checkpoint_path": text_encoder2_path,
+ "emb_size": 768,
+ "max_length": 77,
+ },
+ }
+
+ conf = {
+ "model": {
+ "checkpoint_path": dit_path,
+ "vae": vae,
+ "text_embedder": text_embedder,
+ "dit_params": dit_params,
+ "attention": attention,
+ "num_steps": 50,
+ "guidance_weight": 3.5,
+ },
+ "metrics": {"scale_factor": (1, 2, 2), "resolution": 1024},
+ }
+
+ return DictConfig(conf)
diff --git a/models/kandinsky5/kandinsky_handler.py b/models/kandinsky5/kandinsky_handler.py
new file mode 100644
index 0000000..8d98d42
--- /dev/null
+++ b/models/kandinsky5/kandinsky_handler.py
@@ -0,0 +1,324 @@
+import os
+import torch
+from omegaconf import OmegaConf
+from shared.utils import files_locator as fl
+
+
+def get_kandinsky_text_encoder_filename(text_encoder_quantization):
+ text_encoder_filename = "Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct_bf16.safetensors"
+ if text_encoder_quantization == "int8":
+ text_encoder_filename = text_encoder_filename.replace("bf16", "quanto_bf16_int8")
+ return fl.locate_file(text_encoder_filename, True)
+
+
+_MAGCACHE_RATIOS_CACHE = {}
+
+
+def _load_magcache_ratios(config_name):
+ ratios = _MAGCACHE_RATIOS_CACHE.get(config_name)
+ if ratios is not None:
+ return ratios
+ config_path = os.path.join("models", "kandinsky5", "configs", config_name)
+ if not os.path.isfile(config_path):
+ _MAGCACHE_RATIOS_CACHE[config_name] = None
+ return None
+ conf = OmegaConf.load(config_path)
+ ratios = None
+ if hasattr(conf, "magcache") and "mag_ratios" in conf.magcache:
+ ratios = list(conf.magcache.mag_ratios)
+ _MAGCACHE_RATIOS_CACHE[config_name] = ratios
+ return ratios
+
+
+def _select_k5_bucket(width, height, is_video):
+ if width is None or height is None:
+ return 512
+ area = width * height
+ if is_video:
+ bucket_areas = {512: 512 * 768, 1024: 1024 * 1024}
+ else:
+ bucket_areas = {512: 512 * 512, 1024: 1024 * 1024}
+ return min(bucket_areas, key=lambda res: abs(area - bucket_areas[res]))
+
+
+def _is_k5_sparse(model_type, model_def):
+ if model_type and "sparse" in model_type.lower():
+ return True
+ overrides = (model_def or {}).get("k5_config_overrides", {})
+ attention = overrides.get("model", {}).get("attention", {})
+ return attention.get("type") == "nabla"
+
+
+def _select_k5_magcache_config(base_model_type, model_type, bucket, is_sparse):
+ bucket_tag = "hd" if bucket == 1024 else "sd"
+ model_type = (model_type or "").lower()
+ if base_model_type == "k5_pro_t2v":
+ if "10s" in model_type:
+ return f"k5_pro_t2v_10s_sft_{bucket_tag}.yaml"
+ return f"k5_pro_t2v_5s_sft_{bucket_tag}.yaml"
+ if base_model_type == "k5_pro_i2v":
+ return f"k5_pro_i2v_5s_sft_{bucket_tag}.yaml"
+ if base_model_type == "k5_lite_t2v":
+ if "10s" in model_type:
+ return "k5_lite_t2v_10s_sft_sd.yaml"
+ return "k5_lite_t2v_5s_sft_sd.yaml"
+ if base_model_type == "k5_lite_i2v":
+ return "k5_lite_i2v_5s_sft_sd.yaml"
+ return None
+
+
+def _infer_task(base_model_type):
+ if not base_model_type:
+ return "t2v"
+ base = base_model_type.lower()
+ if "i2v" in base:
+ return "i2v"
+ if "t2v" in base:
+ return "t2v"
+ if "i2i" in base:
+ return "i2i"
+ if "t2i" in base:
+ return "t2i"
+ return "t2v"
+
+
+class family_handler:
+ @staticmethod
+ def query_supported_types():
+ return [
+ "k5_lite_t2v",
+ "k5_lite_i2v",
+ "k5_pro_t2v",
+ "k5_pro_i2v",
+ ]
+
+ @staticmethod
+ def query_family_maps():
+ return {}, {}
+
+ @staticmethod
+ def query_model_family():
+ return "kandinsky5"
+
+ @staticmethod
+ def query_family_infos():
+ return {
+ "kandinsky5": (60, "Kandinsky 5"),
+ }
+
+ @staticmethod
+ def register_lora_cli_args(parser):
+ parser.add_argument(
+ "--lora-dir-kandinsky5",
+ type=str,
+ default=os.path.join("loras", "kandinsky5"),
+ help="Base path for Kandinsky 5 loras (per-architecture subfolders are used).",
+ )
+ parser.add_argument(
+ "--lora-dir-k5-lite-t2v",
+ type=str,
+ default=os.path.join("loras", "k5_lite_t2v"),
+ help="Path to a directory that contains Kandinsky 5 Lite T2V loras.",
+ )
+ parser.add_argument(
+ "--lora-dir-k5-lite-i2v",
+ type=str,
+ default=os.path.join("loras", "k5_lite_i2v"),
+ help="Path to a directory that contains Kandinsky 5 Lite I2V loras.",
+ )
+ parser.add_argument(
+ "--lora-dir-k5-pro-t2v",
+ type=str,
+ default=os.path.join("loras", "k5_pro_t2v"),
+ help="Path to a directory that contains Kandinsky 5 Pro T2V loras.",
+ )
+ parser.add_argument(
+ "--lora-dir-k5-pro-i2v",
+ type=str,
+ default=os.path.join("loras", "k5_pro_i2v"),
+ help="Path to a directory that contains Kandinsky 5 Pro I2V loras.",
+ )
+
+ @staticmethod
+ def get_lora_dir(base_model_type, args):
+ base_dir = getattr(args, "lora_dir_kandinsky5", None) or os.path.join("loras", "kandinsky5")
+ per_arch = {
+ "k5_lite_t2v": getattr(args, "lora_dir_k5_lite_t2v", None) or os.path.join("loras", "k5_lite_t2v"),
+ "k5_lite_i2v": getattr(args, "lora_dir_k5_lite_i2v", None) or os.path.join("loras", "k5_lite_i2v"),
+ "k5_pro_t2v": getattr(args, "lora_dir_k5_pro_t2v", None) or os.path.join("loras", "k5_pro_t2v"),
+ "k5_pro_i2v": getattr(args, "lora_dir_k5_pro_i2v", None) or os.path.join("loras", "k5_pro_i2v"),
+ }
+ if base_model_type in per_arch:
+ return per_arch[base_model_type]
+ if base_model_type and base_model_type != "kandinsky5":
+ return os.path.join(base_dir, base_model_type)
+ return base_dir
+
+ @staticmethod
+ def set_cache_parameters(cache_type, base_model_type, model_def, inputs, skip_steps_cache):
+ if cache_type != "mag":
+ return
+ skip_steps_cache.update({
+ "magcache_thresh": 0,
+ "magcache_K": 2,
+ })
+ resolution = inputs.get("resolution")
+ width = height = None
+ if isinstance(resolution, str) and "x" in resolution:
+ width_str, height_str = resolution.split("x", 1)
+ if width_str.isdigit() and height_str.isdigit():
+ width = int(width_str)
+ height = int(height_str)
+ bucket = _select_k5_bucket(width, height, is_video=True)
+ model_type = inputs.get("model_type")
+ is_sparse = _is_k5_sparse(model_type, model_def)
+ config_name = _select_k5_magcache_config(base_model_type, model_type, bucket, is_sparse)
+ if not config_name:
+ return
+ ratios = _load_magcache_ratios(config_name)
+ if ratios:
+ skip_steps_cache.def_mag_ratios = ratios
+
+ @staticmethod
+ def query_model_def(base_model_type, model_def):
+ task = _infer_task(base_model_type)
+ is_video = task in ("t2v", "i2v")
+ is_image = task in ("t2i", "i2i")
+
+ profiles_dir = base_model_type or "kandinsky5"
+ extra_model_def = {
+ "i2v_class": task == "i2v",
+ "t2v_class": task == "t2v",
+ "image_outputs": is_image,
+ "guidance_max_phases": 1,
+ "sliding_window": False,
+ "flow_shift": True,
+ "mag_cache": True,
+ "profiles_dir": [profiles_dir],
+ }
+
+ if is_video:
+ extra_model_def.update(
+ {
+ "fps": 24,
+ "frames_minimum": 5,
+ "frames_steps": 4,
+ }
+ )
+ else:
+ extra_model_def.update(
+ {
+ "fps": 1,
+ "frames_minimum": 1,
+ "frames_steps": 1,
+ }
+ )
+
+ if task in ("i2v", "i2i"):
+ extra_model_def["image_prompt_types_allowed"] = "S"
+ else:
+ extra_model_def["image_prompt_types_allowed"] = ""
+
+ return extra_model_def
+
+ @staticmethod
+ def query_model_files(computeList, base_model_type, model_filename, text_encoder_quantization):
+ text_encoder_filename = get_kandinsky_text_encoder_filename(text_encoder_quantization)
+ return [
+ {
+ "repoId": "DeepBeepMeep/Qwen_image",
+ "sourceFolderList": ["", "Qwen2.5-VL-7B-Instruct"],
+ "fileList": [
+ ["qwen_vae.safetensors", "qwen_vae_config.json"],
+ [
+ "merges.txt",
+ "tokenizer_config.json",
+ "config.json",
+ "vocab.json",
+ "video_preprocessor_config.json",
+ "preprocessor_config.json",
+ "chat_template.json",
+ ]
+ + computeList(text_encoder_filename),
+ ],
+ },
+ {
+ "repoId": "DeepBeepMeep/HunyuanVideo",
+ "sourceFolderList": ["clip_vit_large_patch14", ""],
+ "fileList": [
+ [
+ "config.json",
+ "merges.txt",
+ "model.safetensors",
+ "preprocessor_config.json",
+ "special_tokens_map.json",
+ "tokenizer.json",
+ "tokenizer_config.json",
+ "vocab.json",
+ ],
+ [
+ "hunyuan_video_VAE_fp32.safetensors",
+ "hunyuan_video_VAE_config.json",
+ ],
+ ],
+ },
+ ]
+
+ @staticmethod
+ def load_model(
+ model_filename,
+ model_type=None,
+ base_model_type=None,
+ model_def=None,
+ quantizeTransformer=False,
+ text_encoder_quantization=None,
+ dtype=torch.bfloat16,
+ VAE_dtype=torch.float32,
+ mixed_precision_transformer=False,
+ save_quantized=False,
+ submodel_no_list=None,
+ override_text_encoder=None,
+ **kwargs,
+ ):
+ from .kandinsky_main import model_factory
+
+ kandinsky = model_factory(
+ checkpoint_dir="ckpts",
+ model_filename=model_filename,
+ model_type=model_type,
+ model_def=model_def,
+ base_model_type=base_model_type,
+ text_encoder_filename=override_text_encoder,
+ quantizeTransformer=quantizeTransformer,
+ dtype=dtype,
+ VAE_dtype=VAE_dtype,
+ mixed_precision_transformer=mixed_precision_transformer,
+ save_quantized=save_quantized,
+ )
+
+ pipe = {
+ "transformer": kandinsky.transformer,
+ "text_encoder": kandinsky.text_embedder.embedder.model,
+ "text_encoder_2": kandinsky.text_embedder.clip_embedder.model,
+ "vae": kandinsky.vae,
+ }
+ for module in pipe.values():
+ if isinstance(module, torch.nn.Module):
+ module.to("cpu")
+ return kandinsky, pipe
+
+ @staticmethod
+ def update_default_settings(base_model_type, model_def, ui_defaults):
+ task = _infer_task(base_model_type)
+ if task in ("t2i", "i2i"):
+ ui_defaults["image_mode"] = 1
+ if task in ("i2v", "i2i"):
+ ui_defaults["image_prompt_type"] = "S"
+
+ ui_defaults["skip_steps_start_step_perc"] = 20
+
+ @staticmethod
+ def get_rgb_factors(base_model_type):
+ from shared.RGB_factors import get_rgb_factors
+
+ return get_rgb_factors("hunyuan")
diff --git a/models/kandinsky5/kandinsky_main.py b/models/kandinsky5/kandinsky_main.py
new file mode 100644
index 0000000..358f422
--- /dev/null
+++ b/models/kandinsky5/kandinsky_main.py
@@ -0,0 +1,265 @@
+import os
+import json
+import torch
+from accelerate import init_empty_weights
+from omegaconf import OmegaConf
+from mmgp import offload
+from shared.utils import files_locator as fl
+from shared.utils.utils import convert_image_to_tensor, convert_tensor_to_image
+from PIL import Image
+
+from .kandinsky.models.dit import get_dit
+from .kandinsky.models.text_embedders import get_text_embedder
+from models.hyvideo.vae.autoencoder_kl_causal_3d import AutoencoderKLCausal3D
+from .kandinsky.pipeline import Kandinsky5Pipeline
+
+
+def _resolve_repo_path(path):
+ if path is None:
+ return None
+ if os.path.isabs(path):
+ return path
+ base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+ return os.path.join(base_dir, path)
+
+
+def _infer_task(base_model_type):
+ if not base_model_type:
+ return "t2v"
+ base = base_model_type.lower()
+ if "i2v" in base:
+ return "i2v"
+ if "t2v" in base:
+ return "t2v"
+ if "i2i" in base:
+ return "i2i"
+ if "t2i" in base:
+ return "t2i"
+ return "t2v"
+
+
+def _get_config_path(base_model_type):
+ if not base_model_type:
+ raise ValueError("Missing base_model_type for Kandinsky config resolution.")
+ return os.path.join("models", "kandinsky5", "configs", f"{base_model_type}.yaml")
+
+
+def _select_qwen_checkpoint(path_hint=None):
+ if path_hint is not None:
+ return path_hint
+ candidates = [
+ os.path.join("Qwen2.5-VL-7B-Instruct", "Qwen2.5-VL-7B-Instruct_bf16.safetensors"),
+ os.path.join("Qwen2.5-VL-7B-Instruct", "Qwen2.5-VL-7B-Instruct_quanto_bf16_int8.safetensors"),
+ ]
+ for candidate in candidates:
+ resolved = fl.locate_file(candidate, error_if_none=False)
+ if resolved is not None:
+ return resolved
+ return fl.locate_folder("Qwen2.5-VL-7B-Instruct")
+
+
+def _preprocess_dit_state_dict(state_dict):
+ prefixes = ["model.diffusion_model.", "diffusion_model.", "model."]
+ for prefix in prefixes:
+ if any(key.startswith(prefix) for key in state_dict):
+ return {key[len(prefix):]: value for key, value in state_dict.items() if key.startswith(prefix)}
+ return state_dict
+
+
+def _apply_vae_tile_config(vae, tile_config):
+ if tile_config is None:
+ if hasattr(vae, "disable_tiling"):
+ vae.disable_tiling()
+ return
+ if hasattr(vae, "apply_tile_config"):
+ vae.apply_tile_config(tile_config)
+ return
+ for key in (
+ "tile_sample_min_tsize",
+ "tile_latent_min_tsize",
+ "tile_sample_min_size",
+ "tile_latent_min_size",
+ "tile_overlap_factor",
+ ):
+ if key in tile_config and hasattr(vae, key):
+ setattr(vae, key, tile_config[key])
+ if hasattr(vae, "enable_tiling"):
+ vae.enable_tiling()
+
+
+class model_factory:
+ def __init__(
+ self,
+ checkpoint_dir,
+ model_filename=None,
+ model_type=None,
+ model_def=None,
+ base_model_type=None,
+ text_encoder_filename=None,
+ quantizeTransformer=False,
+ save_quantized=False,
+ dtype=torch.bfloat16,
+ VAE_dtype=torch.float32,
+ mixed_precision_transformer=False,
+ **kwargs,
+ ):
+ self.device = torch.device("cuda")
+ load_device = torch.device("cpu")
+ self.dtype = dtype
+ self.VAE_dtype = VAE_dtype
+ self.model_def = model_def or {}
+ self.base_model_type = base_model_type or self.model_def.get("architecture") or model_type
+ self.mode = _infer_task(self.base_model_type)
+
+ config_path = _resolve_repo_path(_get_config_path(self.base_model_type))
+ conf = OmegaConf.load(config_path)
+ overrides = self.model_def.get("k5_config_overrides")
+ if overrides:
+ conf = OmegaConf.merge(conf, OmegaConf.create(overrides))
+
+ if isinstance(model_filename, (list, tuple)):
+ model_filename = model_filename[0]
+ conf.model.checkpoint_path = model_filename
+
+ qwen_path = _select_qwen_checkpoint(text_encoder_filename)
+ conf.model.text_embedder.qwen.checkpoint_path = qwen_path
+ conf.model.text_embedder.clip.checkpoint_path = fl.locate_folder("clip_vit_large_patch14")
+
+ vae_filename = fl.locate_file("hunyuan_video_VAE_fp32.safetensors")
+ vae_config = fl.locate_file("hunyuan_video_VAE_config.json")
+ conf.model.vae.checkpoint_path = vae_filename
+ conf.model.vae.name = "hunyuan"
+ conf.model.vae.config_path = vae_config
+
+ with init_empty_weights():
+ dit = get_dit(conf.model.dit_params)
+ offload.load_model_data(
+ dit,
+ model_filename,
+ writable_tensors=False,
+ preprocess_sd=_preprocess_dit_state_dict,
+ )
+ offload.change_dtype(dit, dtype, True)
+ dit.eval().requires_grad_(False)
+ if save_quantized:
+ from wgp import save_quantized_model
+ save_quantized_model(dit, model_type, model_filename, dtype, None)
+
+ quantized_qwen = isinstance(qwen_path, str) and "quanto" in qwen_path.lower()
+ text_embedder = get_text_embedder(
+ conf.model.text_embedder, device=load_device, quantized_qwen=quantized_qwen
+ )
+
+ vae = offload.fast_load_transformers_model(
+ vae_filename,
+ writable_tensors=True,
+ modelClass=AutoencoderKLCausal3D,
+ defaultConfigPath=vae_config,
+ default_dtype=VAE_dtype,
+ )
+ vae = vae.to(dtype=VAE_dtype, device=load_device).eval()
+
+ device_map = {"dit": self.device, "vae": self.device, "text_embedder": self.device}
+ self.pipeline = Kandinsky5Pipeline(
+ mode=self.mode,
+ device_map=device_map,
+ dit=dit,
+ text_embedder=text_embedder,
+ vae=vae,
+ conf=conf,
+ )
+ self._interrupt = False
+
+ self.transformer = dit
+ self.text_embedder = text_embedder
+ self.vae = vae
+
+ def generate(
+ self,
+ seed: int | None = None,
+ input_prompt: str = "",
+ n_prompt: str = "",
+ sampling_steps: int = 30,
+ guide_scale: float = 5.0,
+ frame_num: int = 81,
+ width: int = 768,
+ height: int = 512,
+ image_start=None,
+ image_mode: int = 0,
+ shift: float = 10.0,
+ callback=None,
+ progress: bool | None = None,
+ joint_pass: bool = False,
+ VAE_tile_size: dict | None = None,
+ **kwargs,
+ ):
+ if seed is not None and seed < 0:
+ seed = None
+ if progress is None:
+ progress = callback is None
+
+ negative_caption = n_prompt or ""
+ scheduler_scale = shift
+
+ if VAE_tile_size is not None:
+ _apply_vae_tile_config(self.vae, VAE_tile_size)
+ self.vae._use_vae_tiling = True
+ else:
+ _apply_vae_tile_config(self.vae, None)
+ self.vae._use_vae_tiling = False
+
+ image_pil = None
+ if image_start is not None:
+ if torch.is_tensor(image_start):
+ image_pil = convert_tensor_to_image(image_start)
+ elif isinstance(image_start, Image.Image):
+ image_pil = image_start
+ else:
+ raise ValueError(f"Unsupported image_start type: {type(image_start)}")
+
+ time_length = 0 if image_mode > 0 else 1
+ outputs = self.pipeline(
+ text=input_prompt,
+ image=image_pil,
+ time_length=time_length,
+ width=width,
+ height=height,
+ frame_num=frame_num,
+ seed=seed,
+ num_steps=sampling_steps,
+ guidance_weight=guide_scale,
+ scheduler_scale=scheduler_scale,
+ negative_caption=negative_caption,
+ expand_prompts=False,
+ save_path=None,
+ progress=progress,
+ callback=callback,
+ joint_pass=joint_pass,
+ )
+ if outputs is None:
+ return None
+
+ if isinstance(outputs, list):
+ frames = [convert_image_to_tensor(frame) for frame in outputs]
+ frames = torch.stack(frames, dim=0)
+ video = frames.permute(1, 0, 2, 3)
+ else:
+ if outputs.dim() == 5:
+ video = outputs[0].float().div_(127.5).sub_(1.0)
+ else:
+ video = outputs.float().div_(127.5).sub_(1.0)
+ return video
+
+ def get_loras_transformer(self, *args, **kwargs):
+ return [], []
+
+ @property
+ def _interrupt(self):
+ if hasattr(self, "pipeline"):
+ return self.pipeline._interrupt
+ return False
+
+ @_interrupt.setter
+ def _interrupt(self, value):
+ if hasattr(self, "pipeline"):
+ self.pipeline._interrupt = value
diff --git a/shared/tools/validate_outputs.py b/shared/tools/validate_outputs.py
new file mode 100644
index 0000000..c639d25
--- /dev/null
+++ b/shared/tools/validate_outputs.py
@@ -0,0 +1,93 @@
+import argparse
+import os
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+
+IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp"}
+VIDEO_EXTS = {".mp4", ".mov", ".webm", ".mkv", ".avi"}
+
+
+def _frame_stats(frame: np.ndarray) -> tuple[float, float]:
+ if frame is None:
+ return 0.0, 0.0
+ gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+ return float(np.std(gray)), float(np.max(gray) - np.min(gray))
+
+
+def _check_image(path: Path, min_std: float, min_range: float) -> tuple[bool, str]:
+ img = cv2.imread(str(path), cv2.IMREAD_COLOR)
+ if img is None:
+ return False, "unable to decode image"
+ std, rng = _frame_stats(img)
+ if std < min_std or rng < min_range:
+ return False, f"low variance (std={std:.2f}, range={rng:.2f})"
+ return True, f"ok (std={std:.2f}, range={rng:.2f})"
+
+
+def _sample_video_frames(cap: cv2.VideoCapture, indices: list[int]) -> list[np.ndarray]:
+ frames = []
+ for idx in indices:
+ cap.set(cv2.CAP_PROP_POS_FRAMES, idx)
+ ok, frame = cap.read()
+ if ok:
+ frames.append(frame)
+ return frames
+
+
+def _check_video(path: Path, min_std: float, min_range: float) -> tuple[bool, str]:
+ cap = cv2.VideoCapture(str(path))
+ if not cap.isOpened():
+ return False, "unable to open video"
+ frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
+ if frame_count <= 0:
+ cap.release()
+ return False, "no frames detected"
+ indices = [0, frame_count // 2, max(frame_count - 1, 0)]
+ frames = _sample_video_frames(cap, indices)
+ cap.release()
+ if not frames:
+ return False, "failed to read sample frames"
+ stats = [_frame_stats(frame) for frame in frames]
+ avg_std = sum(s[0] for s in stats) / len(stats)
+ avg_rng = sum(s[1] for s in stats) / len(stats)
+ if avg_std < min_std or avg_rng < min_range:
+ return False, f"low variance (avg std={avg_std:.2f}, avg range={avg_rng:.2f})"
+ return True, f"ok (avg std={avg_std:.2f}, avg range={avg_rng:.2f})"
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="Validate generated outputs are not garbage.")
+ parser.add_argument("output_dir", help="Output directory to scan.")
+ parser.add_argument("--min-std", type=float, default=2.0, help="Minimum grayscale stddev.")
+ parser.add_argument("--min-range", type=float, default=10.0, help="Minimum grayscale range.")
+ args = parser.parse_args()
+
+ root = Path(args.output_dir)
+ if not root.exists():
+ raise SystemExit(f"Output directory not found: {root}")
+
+ files = [p for p in root.rglob("*") if p.suffix.lower() in IMAGE_EXTS.union(VIDEO_EXTS)]
+ if not files:
+ raise SystemExit(f"No output media files found in {root}")
+
+ failures = []
+ for path in sorted(files):
+ if path.suffix.lower() in IMAGE_EXTS:
+ ok, detail = _check_image(path, args.min_std, args.min_range)
+ else:
+ ok, detail = _check_video(path, args.min_std, args.min_range)
+ status = "ok" if ok else "fail"
+ print(f"[{status}] {path}: {detail}")
+ if not ok:
+ failures.append(path)
+
+ if failures:
+ raise SystemExit(f"{len(failures)} file(s) failed validation.")
+ print(f"Validated {len(files)} file(s).")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/wgp.py b/wgp.py
index bc88f23..dd1ae77 100644
--- a/wgp.py
+++ b/wgp.py
@@ -86,7 +86,7 @@ AUTOSAVE_TEMPLATE_PATH = AUTOSAVE_FILENAME
CONFIG_FILENAME = "wgp_config.json"
PROMPT_VARS_MAX = 10
target_mmgp_version = "3.6.9"
-WanGP_version = "9.9"
+WanGP_version = "9.91"
settings_version = 2.41
max_source_video_frames = 3000
prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None
@@ -1325,12 +1325,23 @@ def _parse_settings_json(filename, state):
with open(filename, 'r', encoding='utf-8') as f:
params = json.load(f)
- if not isinstance(params, dict):
- return None, "Settings file must contain a JSON object"
-
- # Wrap as single-task manifest
- task_id += 1
- manifest = [{"id": task_id, "params": params, "plugin_data": {}}]
+ if isinstance(params, list):
+ # Accept full queue manifests or a list of settings dicts
+ if all(isinstance(item, dict) and "params" in item for item in params):
+ manifest = params
+ else:
+ manifest = []
+ for item in params:
+ if not isinstance(item, dict):
+ continue
+ task_id += 1
+ manifest.append({"id": task_id, "params": item, "plugin_data": {}})
+ elif isinstance(params, dict):
+ # Wrap as single-task manifest
+ task_id += 1
+ manifest = [{"id": task_id, "params": params, "plugin_data": {}}]
+ else:
+ return None, "Settings file must contain a JSON object or a list of tasks"
# Media paths are relative to WanGP folder (no cache needed)
wgp_folder = os.path.dirname(os.path.abspath(__file__))
@@ -1665,7 +1676,7 @@ def update_generation_status(html_content):
if(html_content):
return gr.update(value=html_content)
-family_handlers = ["models.wan.wan_handler", "models.wan.ovi_handler", "models.wan.df_handler", "models.hyvideo.hunyuan_handler", "models.ltx_video.ltxv_handler", "models.flux.flux_handler", "models.qwen.qwen_handler", "models.z_image.z_image_handler", "models.chatterbox.chatterbox_handler"]
+family_handlers = ["models.wan.wan_handler", "models.wan.ovi_handler", "models.wan.df_handler", "models.hyvideo.hunyuan_handler", "models.ltx_video.ltxv_handler", "models.flux.flux_handler", "models.qwen.qwen_handler", "models.kandinsky5.kandinsky_handler", "models.z_image.z_image_handler", "models.chatterbox.chatterbox_handler"]
def register_family_lora_args(parser):
registered_families = set()
@@ -1738,6 +1749,11 @@ def _parse_args():
action="store_true",
help="Save a quantized version of the current model"
)
+ parser.add_argument(
+ "--test",
+ action="store_true",
+ help="Load the model and exit generation immediately"
+ )
parser.add_argument(
"--preload",
@@ -2250,6 +2266,8 @@ def test_class_t2v(model_type):
def test_any_sliding_window(model_type):
model_def = get_model_def(model_type)
+ if model_def is None:
+ return False
return model_def.get("sliding_window", False)
def get_model_min_frames_and_step(model_type):
@@ -5111,6 +5129,9 @@ def generate_video(
wan_model, offloadobj = load_models(model_type, override_profile, **model_kwargs)
send_cmd("status", "Model loaded")
reload_needed= False
+ if args.test:
+ send_cmd("info", "Test mode: model loaded, skipping generation.")
+ return
overridden_attention = get_overridden_attention(model_type)
# if overridden_attention is not None and overridden_attention != attention_mode: print(f"Attention mode has been overriden to {overridden_attention} for model type '{model_type}'")
attn = overridden_attention if overridden_attention is not None else attention_mode
@@ -6355,10 +6376,15 @@ def validate_task(task, state):
print(" [SKIP] No model_type specified")
return None
- inputs = {**params, 'prompt': task.get('prompt', '')}
+ inputs = primary_settings.copy()
+ inputs.update(params)
+ inputs['prompt'] = task.get('prompt', '')
+ inputs.setdefault('mode', "")
override_inputs, _, _, _ = validate_settings(state, model_type, single_prompt=True, inputs=inputs)
- if override_inputs is None: return None
- return {**params, **override_inputs}
+ if override_inputs is None:
+ return None
+ inputs.update(override_inputs)
+ return inputs
def process_tasks_cli(queue, state):