Official repository for LTX-Video
Python diffusion-modelsditimage-to-videoimage-to-video-generationtext-to-videotext-to-video-generation
Loading: Support loading checkpoints from Diffusers
Yoav HaCohen16a883dfd253
diff --git a/README.md b/README.mdindex 5ac37e3..40b8b7f 100644--- a/README.md+++ b/README.md@@ -63,10 +63,10 @@ python -m pip install -e .\[inference-script\]Then, download the model from [Hugging Face](https://huggingface.co/Lightricks/LTX-Video)```python-from huggingface_hub import snapshot_download+from huggingface_hub import hf_hub_downloadmodel_path = 'PATH' # The local directory to save downloaded checkpoint-snapshot_download("Lightricks/LTX-Video", local_dir=model_path, local_dir_use_symlinks=False, repo_type='model')+hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltx-video-2b-v0.9.safetensors", local_dir=model_path, local_dir_use_symlinks=False, repo_type='model')```### Inferencediff --git a/inference.py b/inference.pyindex 5e111cd..c3e6617 100644--- a/inference.py+++ b/inference.py@@ -1,5 +1,4 @@import argparse-import jsonimport osimport randomfrom datetime import datetime@@ -8,7 +7,6 @@ from diffusers.utils import loggingimport imageioimport numpy as np-from safetensors import safe_openimport torchimport torch.nn.functional as Ffrom PIL import Image@@ -29,34 +27,6 @@ MAX_WIDTH = 1280MAX_NUM_FRAMES = 257-def load_vae(vae_config, ckpt):- vae = CausalVideoAutoencoder.from_config(vae_config)- vae_state_dict = {- key.replace("vae.", ""): value- for key, value in ckpt.items()- if key.startswith("vae.")- }- vae.load_state_dict(vae_state_dict)- if torch.cuda.is_available():- vae = vae.cuda()- return vae.to(torch.bfloat16)---def load_transformer(transformer_config, ckpt):- transformer = Transformer3DModel.from_config(transformer_config)- transformer_state_dict = {- key.replace("model.diffusion_model.", ""): value- for key, value in ckpt.items()- if key.startswith("model.diffusion_model.")- }- transformer.load_state_dict(transformer_state_dict, strict=True)- if torch.cuda.is_available():- transformer = transformer.cuda()- return transformer---def load_scheduler(scheduler_config):- return RectifiedFlowScheduler.from_config(scheduler_config)def load_image_to_tensor_with_resize_and_crop(@@ -233,9 +203,10 @@ def main():)parser.add_argument(- "--bfloat16",- action="store_true",- help="Denoise in bfloat16",+ "--precision",+ choices=["bfloat16", "mixed_precision"],+ default="bfloat16",+ help="Sets the precision for the transformer and tokenizer. Default is bfloat16. If 'mixed_precision' is enabled, it moves to mixed-precision.",)# Prompts@@ -301,35 +272,28 @@ def main():else:media_items = None- # Paths for the separate mode directoriesckpt_path = Path(args.ckpt_path)- ckpt = {}- with safe_open(ckpt_path, framework="pt", device="cpu") as f:- metadata = f.metadata()- for k in f.keys():- ckpt[k] = f.get_tensor(k)-- configs = json.loads(metadata["config"])- vae_config = configs["vae"]- transformer_config = configs["transformer"]- scheduler_config = configs["scheduler"]-- # Load models- vae = load_vae(vae_config, ckpt)- transformer = load_transformer(transformer_config, ckpt)- scheduler = load_scheduler(scheduler_config)- patchifier = SymmetricPatchifier(patch_size=1)+ vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)+ transformer = Transformer3DModel.from_pretrained(ckpt_path)+ scheduler = RectifiedFlowScheduler.from_pretrained(ckpt_path)+text_encoder = T5EncoderModel.from_pretrained("PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="text_encoder")- if torch.cuda.is_available():- text_encoder = text_encoder.to("cuda")+ patchifier = SymmetricPatchifier(patch_size=1)tokenizer = T5Tokenizer.from_pretrained("PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="tokenizer")- if args.bfloat16 and transformer.dtype != torch.bfloat16:+ if torch.cuda.is_available():+ transformer = transformer.cuda()+ vae = vae.cuda()+ text_encoder = text_encoder.cuda()++ vae = vae.to(torch.bfloat16)+ if args.precision == "bfloat16" and transformer.dtype != torch.bfloat16:transformer = transformer.to(torch.bfloat16)+ text_encoder = text_encoder.to(torch.bfloat16)# Use submodels for the pipelinesubmodel_dict = {@@ -378,7 +342,7 @@ def main():else ConditioningMethod.UNCONDITIONAL),image_cond_noise_scale=args.image_cond_noise_scale,- mixed_precision=not args.bfloat16,+ mixed_precision=(args.precision == "mixed_precision"),).images# Crop the padded images to the desired resolution and number of framesdiff --git a/ltx_video/models/autoencoders/causal_video_autoencoder.py b/ltx_video/models/autoencoders/causal_video_autoencoder.pyindex 80c5f2e..0efafa2 100644--- a/ltx_video/models/autoencoders/causal_video_autoencoder.py+++ b/ltx_video/models/autoencoders/causal_video_autoencoder.py@@ -3,6 +3,7 @@ import osfrom functools import partialfrom types import SimpleNamespacefrom typing import Any, Mapping, Optional, Tuple, Union, List+from pathlib import Pathimport torchimport numpy as np@@ -11,13 +12,20 @@ from torch import nnfrom diffusers.utils import loggingimport torch.nn.functional as Ffrom diffusers.models.embeddings import PixArtAlphaCombinedTimestepSizeEmbeddings+from safetensors import safe_openfrom ltx_video.models.autoencoders.conv_nd_factory import make_conv_nd, make_linear_ndfrom ltx_video.models.autoencoders.pixel_norm import PixelNormfrom ltx_video.models.autoencoders.vae import AutoencoderKLWrapperfrom ltx_video.models.transformers.attention import Attention+from ltx_video.utils.diffusers_config_mapping import (+ diffusers_and_ours_config_mapping,+ make_hashable_key,+ VAE_KEYS_RENAME_DICT,+)+PER_CHANNEL_STATISTICS_PREFIX = "per_channel_statistics."logger = logging.get_logger(__name__) # pylint: disable=invalid-name@@ -29,34 +37,85 @@ class CausalVideoAutoencoder(AutoencoderKLWrapper):*args,**kwargs,):- config_local_path = pretrained_model_name_or_path / "config.json"- config = cls.load_config(config_local_path, **kwargs)- video_vae = cls.from_config(config)- video_vae.to(kwargs["torch_dtype"])+ pretrained_model_name_or_path = Path(pretrained_model_name_or_path)+ if (+ pretrained_model_name_or_path.is_dir()+ and (pretrained_model_name_or_path / "autoencoder.pth").exists()+ ):+ config_local_path = pretrained_model_name_or_path / "config.json"+ config = cls.load_config(config_local_path, **kwargs)++ model_local_path = pretrained_model_name_or_path / "autoencoder.pth"+ state_dict = torch.load(model_local_path, map_location=torch.device("cpu"))++ statistics_local_path = (+ pretrained_model_name_or_path / "per_channel_statistics.json"+ )+ if statistics_local_path.exists():+ with open(statistics_local_path, "r") as file:+ data = json.load(file)+ transposed_data = list(zip(*data["data"]))+ data_dict = {+ col: torch.tensor(vals)+ for col, vals in zip(data["columns"], transposed_data)+ }+ std_of_means = data_dict["std-of-means"]+ mean_of_means = data_dict.get(+ "mean-of-means", torch.zeros_like(data_dict["std-of-means"])+ )+ state_dict[f"{PER_CHANNEL_STATISTICS_PREFIX}std-of-means"] = (+ std_of_means+ )+ state_dict[f"{PER_CHANNEL_STATISTICS_PREFIX}mean-of-means"] = (+ mean_of_means+ )- model_local_path = pretrained_model_name_or_path / "autoencoder.pth"- ckpt_state_dict = torch.load(model_local_path, map_location=torch.device("cpu"))- video_vae.load_state_dict(ckpt_state_dict)+ elif pretrained_model_name_or_path.is_dir():+ config_path = pretrained_model_name_or_path / "vae" / "config.json"+ with open(config_path, "r") as f:+ config = make_hashable_key(json.load(f))- statistics_local_path = (- pretrained_model_name_or_path / "per_channel_statistics.json"- )- if statistics_local_path.exists():- with open(statistics_local_path, "r") as file:- data = json.load(file)- transposed_data = list(zip(*data["data"]))- data_dict = {- col: torch.tensor(vals)- for col, vals in zip(data["columns"], transposed_data)- }- video_vae.register_buffer("std_of_means", data_dict["std-of-means"])- video_vae.register_buffer(- "mean_of_means",- data_dict.get(- "mean-of-means", torch.zeros_like(data_dict["std-of-means"])- ),+ assert config in diffusers_and_ours_config_mapping, (+ "Provided diffusers checkpoint config for VAE is not suppported. "+ "We only support diffusers configs found in Lightricks/LTX-Video.")+ config = diffusers_and_ours_config_mapping[config]++ state_dict_path = (+ pretrained_model_name_or_path+ / "vae"+ / "diffusion_pytorch_model.safetensors"+ )++ state_dict = {}+ with safe_open(state_dict_path, framework="pt", device="cpu") as f:+ for k in f.keys():+ state_dict[k] = f.get_tensor(k)+ for key in list(state_dict.keys()):+ new_key = key+ for replace_key, rename_key in VAE_KEYS_RENAME_DICT.items():+ new_key = new_key.replace(replace_key, rename_key)++ state_dict[new_key] = state_dict.pop(key)++ elif pretrained_model_name_or_path.is_file() and str(+ pretrained_model_name_or_path+ ).endswith(".safetensors"):+ state_dict = {}+ with safe_open(+ pretrained_model_name_or_path, framework="pt", device="cpu"+ ) as f:+ metadata = f.metadata()+ for k in f.keys():+ state_dict[k] = f.get_tensor(k)+ configs = json.loads(metadata["config"])+ config = configs["vae"]++ video_vae = cls.from_config(config)+ if "torch_dtype" in kwargs:+ video_vae.to(kwargs["torch_dtype"])+ video_vae.load_state_dict(state_dict)return video_vae@staticmethod@@ -165,11 +224,16 @@ class CausalVideoAutoencoder(AutoencoderKLWrapper):return json.dumps(self.config.__dict__)def load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True):- per_channel_statistics_prefix = "per_channel_statistics."+ if any([key.startswith("vae.") for key in state_dict.keys()]):+ state_dict = {+ key.replace("vae.", ""): value+ for key, value in state_dict.items()+ if key.startswith("vae.")+ }ckpt_state_dict = {key: valuefor key, value in state_dict.items()- if not key.startswith(per_channel_statistics_prefix)+ if not key.startswith(PER_CHANNEL_STATISTICS_PREFIX)}model_keys = set(name for name, _ in self.named_parameters())@@ -195,9 +259,9 @@ class CausalVideoAutoencoder(AutoencoderKLWrapper):super().load_state_dict(converted_state_dict, strict=strict)data_dict = {- key.removeprefix(per_channel_statistics_prefix): value+ key.removeprefix(PER_CHANNEL_STATISTICS_PREFIX): valuefor key, value in state_dict.items()- if key.startswith(per_channel_statistics_prefix)+ if key.startswith(PER_CHANNEL_STATISTICS_PREFIX)}if len(data_dict) > 0:self.register_buffer("std_of_means", data_dict["std-of-means"])diff --git a/ltx_video/models/transformers/transformer3d.py b/ltx_video/models/transformers/transformer3d.pyindex dfab43d..b53d119 100644--- a/ltx_video/models/transformers/transformer3d.py+++ b/ltx_video/models/transformers/transformer3d.py@@ -1,7 +1,11 @@# Adapted from: https://github.com/huggingface/diffusers/blob/v0.26.3/src/diffusers/models/transformers/transformer_2d.pyimport mathfrom dataclasses import dataclass-from typing import Any, Dict, List, Optional, Literal+from typing import Any, Dict, List, Optional, Literal, Union+import os+import json+import glob+from pathlib import Pathimport torchfrom diffusers.configuration_utils import ConfigMixin, register_to_config@@ -11,6 +15,8 @@ from diffusers.models.normalization import AdaLayerNormSinglefrom diffusers.utils import BaseOutput, is_torch_versionfrom diffusers.utils import loggingfrom torch import nn+from safetensors import safe_open+from ltx_video.models.transformers.attention import BasicTransformerBlockfrom ltx_video.models.transformers.embeddings import get_3d_sincos_pos_embed@@ -305,6 +311,75 @@ class Transformer3DModel(ModelMixin, ConfigMixin):sin_freq = torch.cat([sin_padding, sin_freq], dim=-1)return cos_freq.to(self.dtype), sin_freq.to(self.dtype)+ def load_state_dict(+ self,+ state_dict: Dict,+ *args,+ **kwargs,+ ):+ if any([key.startswith("model.diffusion_model.") for key in state_dict.keys()]):+ state_dict = {+ key.replace("model.diffusion_model.", ""): value+ for key, value in state_dict.items()+ if key.startswith("model.diffusion_model.")+ }+ super().load_state_dict(state_dict, **kwargs)++ @classmethod+ def from_pretrained(+ cls,+ pretrained_model_path: Optional[Union[str, os.PathLike]],+ *args,+ **kwargs,+ ):+ pretrained_model_path = Path(pretrained_model_path)+ if pretrained_model_path.is_dir():+ config_path = pretrained_model_path / "transformer" / "config.json"+ with open(config_path, "r") as f:+ config = make_hashable_key(json.load(f))++ assert config in diffusers_and_ours_config_mapping, (+ "Provided diffusers checkpoint config for transformer is not suppported. "+ "We only support diffusers configs found in Lightricks/LTX-Video."+ )++ config = diffusers_and_ours_config_mapping[config]+ state_dict = {}+ ckpt_paths = (+ pretrained_model_path+ / "transformer"+ / "diffusion_pytorch_model*.safetensors"+ )+ dict_list = glob.glob(str(ckpt_paths))+ for dict_path in dict_list:+ part_dict = {}+ with safe_open(dict_path, framework="pt", device="cpu") as f:+ for k in f.keys():+ part_dict[k] = f.get_tensor(k)+ state_dict.update(part_dict)++ for key in list(state_dict.keys()):+ new_key = key+ for replace_key, rename_key in TRANSFORMER_KEYS_RENAME_DICT.items():+ new_key = new_key.replace(replace_key, rename_key)+ state_dict[new_key] = state_dict.pop(key)++ transformer = cls.from_config(config)+ transformer.load_state_dict(state_dict, strict=True)+ elif pretrained_model_path.is_file() and str(pretrained_model_path).endswith(+ ".safetensors"+ ):+ comfy_single_file_state_dict = {}+ with safe_open(pretrained_model_path, framework="pt", device="cpu") as f:+ metadata = f.metadata()+ for k in f.keys():+ comfy_single_file_state_dict[k] = f.get_tensor(k)+ configs = json.loads(metadata["config"])+ transformer_config = configs["transformer"]+ transformer = Transformer3DModel.from_config(transformer_config)+ transformer.load_state_dict(comfy_single_file_state_dict)+ return transformer+def forward(self,hidden_states: torch.Tensor,diff --git a/ltx_video/schedulers/rf.py b/ltx_video/schedulers/rf.pyindex 68929b0..892266d 100644--- a/ltx_video/schedulers/rf.py+++ b/ltx_video/schedulers/rf.py@@ -2,15 +2,25 @@ import mathfrom abc import ABC, abstractmethodfrom dataclasses import dataclassfrom typing import Callable, Optional, Tuple, Union+import json+import os+from pathlib import Pathimport torchfrom diffusers.configuration_utils import ConfigMixin, register_to_configfrom diffusers.schedulers.scheduling_utils import SchedulerMixinfrom diffusers.utils import BaseOutputfrom torch import Tensor+from safetensors import safe_open+from ltx_video.utils.torch_utils import append_dims+from ltx_video.utils.diffusers_config_mapping import (+ diffusers_and_ours_config_mapping,+ make_hashable_key,+)+def simple_diffusion_resolution_dependent_timestep_shift(samples: Tensor,@@ -197,6 +207,31 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):self.num_inference_steps = num_inference_stepsself.sigmas = self.timesteps+ @staticmethod+ def from_pretrained(pretrained_model_path: Union[str, os.PathLike]):+ pretrained_model_path = Path(pretrained_model_path)+ if pretrained_model_path.is_file():+ comfy_single_file_state_dict = {}+ with safe_open(pretrained_model_path, framework="pt", device="cpu") as f:+ metadata = f.metadata()+ for k in f.keys():+ comfy_single_file_state_dict[k] = f.get_tensor(k)+ configs = json.loads(metadata["config"])+ config = configs["scheduler"]+ del comfy_single_file_state_dict++ elif pretrained_model_path.is_dir():+ diffusers_noise_scheduler_config_path = (+ pretrained_model_path / "scheduler" / "scheduler_config.json"+ )++ with open(diffusers_noise_scheduler_config_path, "r") as f:+ scheduler_config = json.load(f)+ hashable_config = make_hashable_key(scheduler_config)+ if hashable_config in diffusers_and_ours_config_mapping:+ config = diffusers_and_ours_config_mapping[hashable_config]+ return RectifiedFlowScheduler.from_config(config)+def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor:diff --git a/ltx_video/utils/diffusers_config_mapping.py b/ltx_video/utils/diffusers_config_mapping.pynew file mode 100644index 0000000..53c0082--- /dev/null+++ b/ltx_video/utils/diffusers_config_mapping.py@@ -0,0 +1,174 @@+def make_hashable_key(dict_key):+ def convert_value(value):+ if isinstance(value, list):+ return tuple(value)+ elif isinstance(value, dict):+ return tuple(sorted((k, convert_value(v)) for k, v in value.items()))+ else:+ return value++ return tuple(sorted((k, convert_value(v)) for k, v in dict_key.items()))+++DIFFUSERS_SCHEDULER_CONFIG = {+ "_class_name": "FlowMatchEulerDiscreteScheduler",+ "_diffusers_version": "0.32.0.dev0",+ "base_image_seq_len": 1024,+ "base_shift": 0.95,+ "invert_sigmas": False,+ "max_image_seq_len": 4096,+ "max_shift": 2.05,+ "num_train_timesteps": 1000,+ "shift": 1.0,+ "shift_terminal": 0.1,+ "use_beta_sigmas": False,+ "use_dynamic_shifting": True,+ "use_exponential_sigmas": False,+ "use_karras_sigmas": False,+}+DIFFUSERS_TRANSFORMER_CONFIG = {+ "_class_name": "LTXVideoTransformer3DModel",+ "_diffusers_version": "0.32.0.dev0",+ "activation_fn": "gelu-approximate",+ "attention_bias": True,+ "attention_head_dim": 64,+ "attention_out_bias": True,+ "caption_channels": 4096,+ "cross_attention_dim": 2048,+ "in_channels": 128,+ "norm_elementwise_affine": False,+ "norm_eps": 1e-06,+ "num_attention_heads": 32,+ "num_layers": 28,+ "out_channels": 128,+ "patch_size": 1,+ "patch_size_t": 1,+ "qk_norm": "rms_norm_across_heads",+}+DIFFUSERS_VAE_CONFIG = {+ "_class_name": "AutoencoderKLLTXVideo",+ "_diffusers_version": "0.32.0.dev0",+ "block_out_channels": [128, 256, 512, 512],+ "decoder_causal": False,+ "encoder_causal": True,+ "in_channels": 3,+ "latent_channels": 128,+ "layers_per_block": [4, 3, 3, 3, 4],+ "out_channels": 3,+ "patch_size": 4,+ "patch_size_t": 1,+ "resnet_norm_eps": 1e-06,+ "scaling_factor": 1.0,+ "spatio_temporal_scaling": [True, True, True, False],+}++OURS_SCHEDULER_CONFIG = {+ "_class_name": "RectifiedFlowScheduler",+ "_diffusers_version": "0.25.1",+ "num_train_timesteps": 1000,+ "shifting": "SD3",+ "base_resolution": None,+ "target_shift_terminal": 0.1,+}++OURS_TRANSFORMER_CONFIG = {+ "_class_name": "Transformer3DModel",+ "_diffusers_version": "0.25.1",+ "_name_or_path": "PixArt-alpha/PixArt-XL-2-256x256",+ "activation_fn": "gelu-approximate",+ "attention_bias": True,+ "attention_head_dim": 64,+ "attention_type": "default",+ "caption_channels": 4096,+ "cross_attention_dim": 2048,+ "double_self_attention": False,+ "dropout": 0.0,+ "in_channels": 128,+ "norm_elementwise_affine": False,+ "norm_eps": 1e-06,+ "norm_num_groups": 32,+ "num_attention_heads": 32,+ "num_embeds_ada_norm": 1000,+ "num_layers": 28,+ "num_vector_embeds": None,+ "only_cross_attention": False,+ "out_channels": 128,+ "project_to_2d_pos": True,+ "upcast_attention": False,+ "use_linear_projection": False,+ "qk_norm": "rms_norm",+ "standardization_norm": "rms_norm",+ "positional_embedding_type": "rope",+ "positional_embedding_theta": 10000.0,+ "positional_embedding_max_pos": [20, 2048, 2048],+ "timestep_scale_multiplier": 1000,+}+OURS_VAE_CONFIG = {+ "_class_name": "CausalVideoAutoencoder",+ "dims": 3,+ "in_channels": 3,+ "out_channels": 3,+ "latent_channels": 128,+ "blocks": [+ ["res_x", 4],+ ["compress_all", 1],+ ["res_x_y", 1],+ ["res_x", 3],+ ["compress_all", 1],+ ["res_x_y", 1],+ ["res_x", 3],+ ["compress_all", 1],+ ["res_x", 3],+ ["res_x", 4],+ ],+ "scaling_factor": 1.0,+ "norm_layer": "pixel_norm",+ "patch_size": 4,+ "latent_log_var": "uniform",+ "use_quant_conv": False,+ "causal_decoder": False,+}+++diffusers_and_ours_config_mapping = {+ make_hashable_key(DIFFUSERS_SCHEDULER_CONFIG): OURS_SCHEDULER_CONFIG,+ make_hashable_key(DIFFUSERS_TRANSFORMER_CONFIG): OURS_TRANSFORMER_CONFIG,+ make_hashable_key(DIFFUSERS_VAE_CONFIG): OURS_VAE_CONFIG,+}+++TRANSFORMER_KEYS_RENAME_DICT = {+ "proj_in": "patchify_proj",+ "time_embed": "adaln_single",+ "norm_q": "q_norm",+ "norm_k": "k_norm",+}+++VAE_KEYS_RENAME_DICT = {+ "decoder.up_blocks.3.conv_in": "decoder.up_blocks.7",+ "decoder.up_blocks.3.upsamplers.0": "decoder.up_blocks.8",+ "decoder.up_blocks.3": "decoder.up_blocks.9",+ "decoder.up_blocks.2.upsamplers.0": "decoder.up_blocks.5",+ "decoder.up_blocks.2.conv_in": "decoder.up_blocks.4",+ "decoder.up_blocks.2": "decoder.up_blocks.6",+ "decoder.up_blocks.1.upsamplers.0": "decoder.up_blocks.2",+ "decoder.up_blocks.1": "decoder.up_blocks.3",+ "decoder.up_blocks.0": "decoder.up_blocks.1",+ "decoder.mid_block": "decoder.up_blocks.0",+ "encoder.down_blocks.3": "encoder.down_blocks.8",+ "encoder.down_blocks.2.downsamplers.0": "encoder.down_blocks.7",+ "encoder.down_blocks.2": "encoder.down_blocks.6",+ "encoder.down_blocks.1.downsamplers.0": "encoder.down_blocks.4",+ "encoder.down_blocks.1.conv_out": "encoder.down_blocks.5",+ "encoder.down_blocks.1": "encoder.down_blocks.3",+ "encoder.down_blocks.0.conv_out": "encoder.down_blocks.2",+ "encoder.down_blocks.0.downsamplers.0": "encoder.down_blocks.1",+ "encoder.down_blocks.0": "encoder.down_blocks.0",+ "encoder.mid_block": "encoder.down_blocks.9",+ "conv_shortcut.conv": "conv_shortcut",+ "resnets": "res_blocks",+ "norm3": "norm3.norm",+ "latents_mean": "per_channel_statistics.mean-of-means",+ "latents_std": "per_channel_statistics.std-of-means",+}