Python
RAM optimisations
DeepBeepMeepf5bf0259359d
diff --git a/README.md b/README.mdindex ad8ed56..e1d9ced 100644--- a/README.md+++ b/README.md@@ -33,7 +33,7 @@ WanGP supports the Wan (and derived models) but also Hunyuan Video, Flux, Qwen,## 🔥 Latest Updates :-### February 10th 2026: WanGP v10.83, Easy Metal+### February 12th 2026: WanGP v10.84, Easy Metal- **Ace Step 1.5 Turbo Super Charged**: all the best features of *Ace Step 1.5* are now in *WanGP* and are *Fast* & *Easy* to use:* Manual Selection of *Bpm*, *Keyscale*, *Time Signature* & *Language*@@ -58,7 +58,8 @@ Improved queue reordering: items can now be dragged and dropped directly onto th*update 10.81*: Fixes\*update 10.82*: UI update\-*update 10.83*: Kugel Audio Split+*update 10.83*: Kugel Audio Split\+*update 10.84*: Ace Step RAM optimizations (fixed memory leak & reduce RAM requirements)**Note to RTX 50xx owners**: you will need to upgrade to *pytorch 2.10* (see upgrade procedure below) to be able to use *Triton*diff --git a/models/TTS/ace_step/music_dcae/music_dcae_pipeline.py b/models/TTS/ace_step/music_dcae/music_dcae_pipeline.pyindex e28f92b..8115687 100644--- a/models/TTS/ace_step/music_dcae/music_dcae_pipeline.py+++ b/models/TTS/ace_step/music_dcae/music_dcae_pipeline.py@@ -42,6 +42,7 @@ class MusicDCAE(ModelMixin, ConfigMixin, FromOriginalModelMixin):dcae_weights_path,modelClass=AutoencoderDC,defaultConfigPath=dcae_config_path,+ writable_tensors=False,default_dtype=self._dtype,ignore_unused_weights=True,)@@ -51,6 +52,7 @@ class MusicDCAE(ModelMixin, ConfigMixin, FromOriginalModelMixin):vocoder_weights_path,modelClass=ADaMoSHiFiGANV1,defaultConfigPath=vocoder_config_path,+ writable_tensors=False,default_dtype=self._dtype,ignore_unused_weights=True,)diff --git a/models/TTS/ace_step/pipeline_ace_step.py b/models/TTS/ace_step/pipeline_ace_step.pyindex 09d44fd..a1c6009 100644--- a/models/TTS/ace_step/pipeline_ace_step.py+++ b/models/TTS/ace_step/pipeline_ace_step.py@@ -158,6 +158,7 @@ class ACEStepPipeline:self.transformer_weights_path,modelClass=ACEStepTransformer2DModel,defaultConfigPath=self.transformer_config_path,+ writable_tensors=False,default_dtype=self.dtype,ignore_unused_weights=True,)@@ -193,6 +194,7 @@ class ACEStepPipeline:self.text_encoder_weights_path,modelClass=UMT5EncoderModel,defaultConfigPath=self.text_encoder_config_path,+ writable_tensors=False,default_dtype=self.dtype,ignore_unused_weights=True,preprocess_sd=_fix_umt5_state_dict,diff --git a/models/TTS/ace_step15/models/autoencoder_oobleck.py b/models/TTS/ace_step15/models/autoencoder_oobleck.pynew file mode 100644index 0000000..299ce97--- /dev/null+++ b/models/TTS/ace_step15/models/autoencoder_oobleck.py@@ -0,0 +1,455 @@+# Copyright 2025 The HuggingFace Team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import math+from dataclasses import dataclass+from typing import Optional, Tuple, Union++import numpy as np+import torch+import torch.nn as nn+from torch.nn.utils.parametrizations import weight_norm++from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models.autoencoders.vae import AutoencoderMixin+from diffusers.models.modeling_utils import ModelMixin+from diffusers.utils import BaseOutput+from diffusers.utils.accelerate_utils import apply_forward_hook+from diffusers.utils.torch_utils import randn_tensor+++class Snake1d(nn.Module):+ """+ A 1-dimensional Snake activation function module.+ """++ def __init__(self, hidden_dim, logscale=True):+ super().__init__()+ self.alpha = nn.Parameter(torch.zeros(1, hidden_dim, 1))+ self.beta = nn.Parameter(torch.zeros(1, hidden_dim, 1))++ self.alpha.requires_grad = True+ self.beta.requires_grad = True+ self.logscale = logscale++ def forward(self, hidden_states, preserve_input: bool = False):+ shape = hidden_states.shape+ hidden_states = hidden_states.reshape(shape[0], shape[1], -1)++ alpha = self.alpha if not self.logscale else torch.exp(self.alpha)+ beta = self.beta if not self.logscale else torch.exp(self.beta)+ inv_beta = (beta + 1e-9).reciprocal()++ use_inplace = (not preserve_input) and (not torch.is_grad_enabled())+ wave = torch.empty_like(hidden_states)+ torch.mul(hidden_states, alpha, out=wave)+ torch.sin_(wave)+ wave.mul_(wave)+ wave.mul_(inv_beta)+ if use_inplace:+ hidden_states.add_(wave)+ return hidden_states.reshape(shape)+ wave.add_(hidden_states)+ return wave.reshape(shape)+++class OobleckResidualUnit(nn.Module):+ """+ A residual unit composed of Snake1d and weight-normalized Conv1d layers with dilations.+ """++ def __init__(self, dimension: int = 16, dilation: int = 1):+ super().__init__()+ pad = ((7 - 1) * dilation) // 2++ self.snake1 = Snake1d(dimension)+ self.conv1 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=7, dilation=dilation, padding=pad))+ self.snake2 = Snake1d(dimension)+ self.conv2 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=1))++ def forward(self, hidden_state):+ """+ Forward pass through the residual unit.++ Args:+ hidden_state (`torch.Tensor` of shape `(batch_size, channels, time_steps)`):+ Input tensor .++ Returns:+ output_tensor (`torch.Tensor` of shape `(batch_size, channels, time_steps)`)+ Input tensor after passing through the residual unit.+ """+ output_tensor = hidden_state+ output_tensor = self.conv1(self.snake1(output_tensor, preserve_input=True))+ output_tensor = self.conv2(self.snake2(output_tensor))++ padding = (hidden_state.shape[-1] - output_tensor.shape[-1]) // 2+ if padding > 0:+ hidden_state = hidden_state[..., padding:-padding]+ output_tensor += hidden_state+ return output_tensor+++class OobleckEncoderBlock(nn.Module):+ """Encoder block used in Oobleck encoder."""++ def __init__(self, input_dim, output_dim, stride: int = 1):+ super().__init__()++ self.res_unit1 = OobleckResidualUnit(input_dim, dilation=1)+ self.res_unit2 = OobleckResidualUnit(input_dim, dilation=3)+ self.res_unit3 = OobleckResidualUnit(input_dim, dilation=9)+ self.snake1 = Snake1d(input_dim)+ self.conv1 = weight_norm(+ nn.Conv1d(input_dim, output_dim, kernel_size=2 * stride, stride=stride, padding=math.ceil(stride / 2))+ )++ def forward(self, hidden_state):+ hidden_state = self.res_unit1(hidden_state)+ hidden_state = self.res_unit2(hidden_state)+ hidden_state = self.snake1(self.res_unit3(hidden_state))+ hidden_state = self.conv1(hidden_state)++ return hidden_state+++class OobleckDecoderBlock(nn.Module):+ """Decoder block used in Oobleck decoder."""++ def __init__(self, input_dim, output_dim, stride: int = 1):+ super().__init__()++ self.snake1 = Snake1d(input_dim)+ self.conv_t1 = weight_norm(+ nn.ConvTranspose1d(+ input_dim,+ output_dim,+ kernel_size=2 * stride,+ stride=stride,+ padding=math.ceil(stride / 2),+ )+ )+ self.res_unit1 = OobleckResidualUnit(output_dim, dilation=1)+ self.res_unit2 = OobleckResidualUnit(output_dim, dilation=3)+ self.res_unit3 = OobleckResidualUnit(output_dim, dilation=9)++ def forward(self, hidden_state):+ hidden_state = self.conv_t1(self.snake1(hidden_state, preserve_input=False))+ hidden_state = self.res_unit1(hidden_state)+ hidden_state = self.res_unit2(hidden_state)+ return self.res_unit3(hidden_state)+++class OobleckDiagonalGaussianDistribution(object):+ def __init__(self, parameters: torch.Tensor, deterministic: bool = False):+ self.parameters = parameters+ self.mean, self.scale = parameters.chunk(2, dim=1)+ self.std = nn.functional.softplus(self.scale) + 1e-4+ self.var = self.std * self.std+ self.logvar = torch.log(self.var)+ self.deterministic = deterministic++ def sample(self, generator: Optional[torch.Generator] = None) -> torch.Tensor:+ # make sure sample is on the same device as the parameters and has same dtype+ sample = randn_tensor(+ self.mean.shape,+ generator=generator,+ device=self.parameters.device,+ dtype=self.parameters.dtype,+ )+ x = self.mean + self.std * sample+ return x++ def kl(self, other: "OobleckDiagonalGaussianDistribution" = None) -> torch.Tensor:+ if self.deterministic:+ return torch.Tensor([0.0])+ else:+ if other is None:+ return (self.mean * self.mean + self.var - self.logvar - 1.0).sum(1).mean()+ else:+ normalized_diff = torch.pow(self.mean - other.mean, 2) / other.var+ var_ratio = self.var / other.var+ logvar_diff = self.logvar - other.logvar++ kl = normalized_diff + var_ratio + logvar_diff - 1++ kl = kl.sum(1).mean()+ return kl++ def mode(self) -> torch.Tensor:+ return self.mean+++@dataclass+class AutoencoderOobleckOutput(BaseOutput):+ """+ Output of AutoencoderOobleck encoding method.++ Args:+ latent_dist (`OobleckDiagonalGaussianDistribution`):+ Encoded outputs of `Encoder` represented as the mean and standard deviation of+ `OobleckDiagonalGaussianDistribution`. `OobleckDiagonalGaussianDistribution` allows for sampling latents+ from the distribution.+ """++ latent_dist: "OobleckDiagonalGaussianDistribution" # noqa: F821+++@dataclass+class OobleckDecoderOutput(BaseOutput):+ r"""+ Output of decoding method.++ Args:+ sample (`torch.Tensor` of shape `(batch_size, audio_channels, sequence_length)`):+ The decoded output sample from the last layer of the model.+ """++ sample: torch.Tensor+++class OobleckEncoder(nn.Module):+ """Oobleck Encoder"""++ def __init__(self, encoder_hidden_size, audio_channels, downsampling_ratios, channel_multiples):+ super().__init__()++ strides = downsampling_ratios+ channel_multiples = [1] + channel_multiples++ # Create first convolution+ self.conv1 = weight_norm(nn.Conv1d(audio_channels, encoder_hidden_size, kernel_size=7, padding=3))++ self.block = []+ # Create EncoderBlocks that double channels as they downsample by `stride`+ for stride_index, stride in enumerate(strides):+ self.block += [+ OobleckEncoderBlock(+ input_dim=encoder_hidden_size * channel_multiples[stride_index],+ output_dim=encoder_hidden_size * channel_multiples[stride_index + 1],+ stride=stride,+ )+ ]++ self.block = nn.ModuleList(self.block)+ d_model = encoder_hidden_size * channel_multiples[-1]+ self.snake1 = Snake1d(d_model)+ self.conv2 = weight_norm(nn.Conv1d(d_model, encoder_hidden_size, kernel_size=3, padding=1))++ def forward(self, hidden_state):+ hidden_state = self.conv1(hidden_state)++ for module in self.block:+ hidden_state = module(hidden_state)++ hidden_state = self.snake1(hidden_state)+ hidden_state = self.conv2(hidden_state)++ return hidden_state+++class OobleckDecoder(nn.Module):+ """Oobleck Decoder"""++ def __init__(self, channels, input_channels, audio_channels, upsampling_ratios, channel_multiples):+ super().__init__()++ strides = upsampling_ratios+ channel_multiples = [1] + channel_multiples++ # Add first conv layer+ self.conv1 = weight_norm(nn.Conv1d(input_channels, channels * channel_multiples[-1], kernel_size=7, padding=3))++ # Add upsampling + MRF blocks+ block = []+ for stride_index, stride in enumerate(strides):+ block += [+ OobleckDecoderBlock(+ input_dim=channels * channel_multiples[len(strides) - stride_index],+ output_dim=channels * channel_multiples[len(strides) - stride_index - 1],+ stride=stride,+ )+ ]++ self.block = nn.ModuleList(block)+ output_dim = channels+ self.snake1 = Snake1d(output_dim)+ self.conv2 = weight_norm(nn.Conv1d(channels, audio_channels, kernel_size=7, padding=3, bias=False))++ def forward(self, hidden_state):+ hidden_state = self.conv1(hidden_state)++ for layer in self.block:+ hidden_state = layer(hidden_state)++ return self.conv2(self.snake1(hidden_state, preserve_input=False))+++class AutoencoderOobleck(ModelMixin, AutoencoderMixin, ConfigMixin):+ r"""+ An autoencoder for encoding waveforms into latents and decoding latent representations into waveforms. First+ introduced in Stable Audio.++ This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented+ for all models (such as downloading or saving).++ Parameters:+ encoder_hidden_size (`int`, *optional*, defaults to 128):+ Intermediate representation dimension for the encoder.+ downsampling_ratios (`List[int]`, *optional*, defaults to `[2, 4, 4, 8, 8]`):+ Ratios for downsampling in the encoder. These are used in reverse order for upsampling in the decoder.+ channel_multiples (`List[int]`, *optional*, defaults to `[1, 2, 4, 8, 16]`):+ Multiples used to determine the hidden sizes of the hidden layers.+ decoder_channels (`int`, *optional*, defaults to 128):+ Intermediate representation dimension for the decoder.+ decoder_input_channels (`int`, *optional*, defaults to 64):+ Input dimension for the decoder. Corresponds to the latent dimension.+ audio_channels (`int`, *optional*, defaults to 2):+ Number of channels in the audio data. Either 1 for mono or 2 for stereo.+ sampling_rate (`int`, *optional*, defaults to 44100):+ The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz).+ """++ _supports_gradient_checkpointing = False+ _supports_group_offloading = False++ @register_to_config+ def __init__(+ self,+ encoder_hidden_size=128,+ downsampling_ratios=[2, 4, 4, 8, 8],+ channel_multiples=[1, 2, 4, 8, 16],+ decoder_channels=128,+ decoder_input_channels=64,+ audio_channels=2,+ sampling_rate=44100,+ ):+ super().__init__()++ self.encoder_hidden_size = encoder_hidden_size+ self.downsampling_ratios = downsampling_ratios+ self.decoder_channels = decoder_channels+ self.upsampling_ratios = downsampling_ratios[::-1]+ self.hop_length = int(np.prod(downsampling_ratios))+ self.sampling_rate = sampling_rate++ self.encoder = OobleckEncoder(+ encoder_hidden_size=encoder_hidden_size,+ audio_channels=audio_channels,+ downsampling_ratios=downsampling_ratios,+ channel_multiples=channel_multiples,+ )++ self.decoder = OobleckDecoder(+ channels=decoder_channels,+ input_channels=decoder_input_channels,+ audio_channels=audio_channels,+ upsampling_ratios=self.upsampling_ratios,+ channel_multiples=channel_multiples,+ )++ self.use_slicing = False++ @apply_forward_hook+ def encode(+ self, x: torch.Tensor, return_dict: bool = True+ ) -> Union[AutoencoderOobleckOutput, Tuple[OobleckDiagonalGaussianDistribution]]:+ """+ Encode a batch of images into latents.++ Args:+ x (`torch.Tensor`): Input batch of images.+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple.++ Returns:+ The latent representations of the encoded images. If `return_dict` is True, a+ [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned.+ """+ if self.use_slicing and x.shape[0] > 1:+ encoded_slices = [self.encoder(x_slice) for x_slice in x.split(1)]+ h = torch.cat(encoded_slices)+ else:+ h = self.encoder(x)++ posterior = OobleckDiagonalGaussianDistribution(h)++ if not return_dict:+ return (posterior,)++ return AutoencoderOobleckOutput(latent_dist=posterior)++ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[OobleckDecoderOutput, torch.Tensor]:+ dec = self.decoder(z)++ if not return_dict:+ return (dec,)++ return OobleckDecoderOutput(sample=dec)++ @apply_forward_hook+ def decode(+ self, z: torch.FloatTensor, return_dict: bool = True, generator=None+ ) -> Union[OobleckDecoderOutput, torch.FloatTensor]:+ """+ Decode a batch of images.++ Args:+ z (`torch.Tensor`): Input batch of latent vectors.+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether to return a [`~models.vae.OobleckDecoderOutput`] instead of a plain tuple.++ Returns:+ [`~models.vae.OobleckDecoderOutput`] or `tuple`:+ If return_dict is True, a [`~models.vae.OobleckDecoderOutput`] is returned, otherwise a plain `tuple`+ is returned.++ """+ if self.use_slicing and z.shape[0] > 1:+ decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)]+ decoded = torch.cat(decoded_slices)+ else:+ decoded = self._decode(z).sample++ if not return_dict:+ return (decoded,)++ return OobleckDecoderOutput(sample=decoded)++ def forward(+ self,+ sample: torch.Tensor,+ sample_posterior: bool = False,+ return_dict: bool = True,+ generator: Optional[torch.Generator] = None,+ ) -> Union[OobleckDecoderOutput, torch.Tensor]:+ r"""+ Args:+ sample (`torch.Tensor`): Input sample.+ sample_posterior (`bool`, *optional*, defaults to `False`):+ Whether to sample from the posterior.+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`OobleckDecoderOutput`] instead of a plain tuple.+ """+ x = sample+ posterior = self.encode(x).latent_dist+ if sample_posterior:+ z = posterior.sample(generator=generator)+ else:+ z = posterior.mode()+ dec = self.decode(z).sample++ if not return_dict:+ return (dec,)++ return OobleckDecoderOutput(sample=dec)diff --git a/models/TTS/ace_step15/pipeline_ace_step15.py b/models/TTS/ace_step15/pipeline_ace_step15.pyindex 3a06320..4bc9e2b 100644--- a/models/TTS/ace_step15/pipeline_ace_step15.py+++ b/models/TTS/ace_step15/pipeline_ace_step15.py@@ -15,11 +15,11 @@ import torchaudioimport yamlfrom tqdm import tqdmfrom transformers import AutoTokenizer, Qwen3ForCausalLM, Qwen3Model-from diffusers import AutoencoderOobleckfrom mmgp import offloadfrom shared.utils.text_encoder_cache import TextEncoderCache+from .models.autoencoder_oobleck import AutoencoderOobleckfrom .models.ace_step15_hf import AceStepConditionGenerationModel_DEFAULT_TIMBRE = [@@ -51,6 +51,7 @@ _ACE_STEP15_DEFAULT_TIMESIGNATURE = 2_ACE_STEP15_DEFAULT_KEYSCALE = "C major"_ACE_STEP15_DURATION_MIN_SECONDS = 5_ACE_STEP15_DURATION_MAX_SECONDS = 600+_ACE_STEP15_PHASE1_MAX_TOKENS = 512_ACE_STEP15_ALLOWED_TIMESIGNATURES = {2, 3, 4, 6}_ACE_STEP15_BPM_OPTIONS = [str(v) for v in range(30, 301)]_ACE_STEP15_TIMESIGNATURE_OPTIONS = ["2", "3", "4", "6"]@@ -111,6 +112,7 @@ class ACEStep15Pipeline:ignore_lm_cache_seed: bool = False,lm_decoder_engine: str = "legacy",lm_vllm_weight_mode: str = "lazy",+ lm_vllm_free_mmap_after_cuda_copy: bool = True,device=None,dtype=torch.bfloat16,):@@ -139,6 +141,11 @@ class ACEStep15Pipeline:if self.lm_engine not in ("legacy", "pt", "vllm"):self.lm_engine = "legacy"self.lm_vllm_weight_mode = (lm_vllm_weight_mode or "lazy").strip().lower()+ if self.lm_engine == "vllm" and self.lm_vllm_weight_mode not in ("lazy", "pinned"):+ raise ValueError(+ f"Unsupported Ace Step vllm LM weight mode '{self.lm_vllm_weight_mode}'. Supported: lazy, pinned."+ )+ self.lm_vllm_free_mmap_after_cuda_copy = bool(lm_vllm_free_mmap_after_cuda_copy)self._interrupt = Falseself._early_stop = False@@ -174,32 +181,45 @@ class ACEStep15Pipeline:modelClass=AceStepConditionGenerationModel,defaultConfigPath=transformer_config_path,default_dtype=self.dtype,+ writable_tensors=False,ignore_unused_weights=True,)self.ace_step_transformer.eval()self.model = self.ace_step_transformer- self._patch_oobleck_weight_norm()self.audio_vae = offload.fast_load_transformers_model(vae_weights_path,modelClass=AutoencoderOobleck,defaultConfigPath=vae_config_path,default_dtype=self.dtype,+ writable_tensors=False,ignore_unused_weights=True,)+ self._fold_oobleck_decoder_weight_norm_once(self.audio_vae)self.audio_vae.eval()self.audio_vae._offload_hooks = ["encode", "decode"]self.audio_vae.get_VAE_tile_size = _ace_step15_get_vae_tile_sizeself.vae = self.audio_vae@staticmethod- def _patch_oobleck_weight_norm():+ def _fold_oobleck_decoder_weight_norm_once(audio_vae):try:- from torch.nn.utils import parametrizations- from diffusers.models.autoencoders import autoencoder_oobleck- autoencoder_oobleck.weight_norm = parametrizations.weight_norm+ from torch.nn.utils import parametrizeexcept Exception:return+ decoder = getattr(audio_vae, "decoder", None)+ if decoder is None:+ return+ folded = 0+ with torch.no_grad():+ for module in decoder.modules():+ module_parametrizations = getattr(module, "parametrizations", None)+ if module_parametrizations is None or "weight" not in module_parametrizations:+ continue+ parametrize.remove_parametrizations(module, "weight", leave_parametrized=True)+ folded += 1+ # if folded > 0:+ # print(f"[ace_step15] Folded Oobleck decoder weight_norm once for {folded} layers.")def _init_lm_hint_modules(self):self._lm_hint_quantizer = None@@ -239,6 +259,7 @@ class ACEStep15Pipeline:self.lm_tokenizer = Noneif not self.enable_lm:return+ print("Loading Tokenizers...")lm_loader = lambda: AutoTokenizer.from_pretrained(self.lm_tokenizer_dir,local_files_only=True,@@ -284,6 +305,7 @@ class ACEStep15Pipeline:modelClass=Qwen3Model,defaultConfigPath=config_path,default_dtype=self.dtype,+ writable_tensors=False,ignore_unused_weights=True,)self.text_encoder_2.eval()@@ -307,6 +329,7 @@ class ACEStep15Pipeline:defaultConfigPath=config_path,default_dtype=self.dtype,preprocess_sd=_remap_lm_state_dict,+ writable_tensors=False,ignore_unused_weights=True,)self.lm_model.eval()@@ -341,6 +364,7 @@ class ACEStep15Pipeline:lm_weights_path=self.lm_weights_path,audio_code_mask=getattr(self, "_audio_code_mask", None),audio_code_token_map=getattr(self, "_audio_code_token_map", {}),+ free_mmap_after_cuda_copy=self.lm_vllm_free_mmap_after_cuda_copy,weight_load_mode=self.lm_vllm_weight_mode,)if self.lm_engine in ("pt", "vllm") and self._lm_engine_impl is None:@@ -614,6 +638,7 @@ class ACEStep15Pipeline:logits_processor_update_state=None,stop_checker=None,progress_label: str = "LM text",+ release_vram_after: bool = True,):if self._lm_engine_impl is not None and hasattr(self._lm_engine_impl, "generate_text"):return self._lm_engine_impl.generate_text(@@ -631,6 +656,7 @@ class ACEStep15Pipeline:logits_processor_update_state=logits_processor_update_state,stop_checker=stop_checker,progress_label=progress_label,+ release_vram_after=release_vram_after,)if self.lm_engine != "legacy":raise RuntimeError(f"LM engine '{self.lm_engine}' is not initialized for text generation.")@@ -779,6 +805,7 @@ class ACEStep15Pipeline:top_p,top_k,callback=None,+ release_vram_after: bool = True,):phase1_metadata = dict(metadata)if self._should_abort():@@ -811,7 +838,7 @@ class ACEStep15Pipeline:text_out = self._generate_lm_text(prompt=prompt,prompt_negative=prompt_negative,- max_tokens=512,+ max_tokens=_ACE_STEP15_PHASE1_MAX_TOKENS,temperature=temperature,top_p=top_p,top_k=top_k,@@ -822,6 +849,7 @@ class ACEStep15Pipeline:logits_processor_update_state=processor.update_state,stop_checker=None,progress_label="LM Compute Metadata",+ release_vram_after=release_vram_after,)if text_out is None:raise RuntimeError("LM phase1 failed while inferring metadata.")@@ -921,6 +949,43 @@ class ACEStep15Pipeline:int(seed_value) if seed_value is not None else None,)+ def _reserve_vllm_runtime_for_max_duration(self, tags: str, lyrics: str, negative_prompt: str, lm_cfg_scale):+ if self.lm_engine != "vllm" or self._lm_engine_impl is None or self.lm_tokenizer is None:+ return+ reserve_fn = getattr(self._lm_engine_impl, "reserve_runtime", None)+ if not callable(reserve_fn):+ return+ cfg_phase1 = 1.0 if lm_cfg_scale is None else float(lm_cfg_scale)+ cfg_phase2 = 2.5 if lm_cfg_scale is None else float(lm_cfg_scale)+ reserve_cfg = max(1.0, cfg_phase1, cfg_phase2)+ max_duration_seconds = int(_ACE_STEP15_DURATION_MAX_SECONDS)+ phase1_max_tokens = int(_ACE_STEP15_PHASE1_MAX_TOKENS)+ phase2_max_tokens = max(1, max_duration_seconds * 5)+ reserve_max_tokens = max(phase1_max_tokens, phase2_max_tokens)++ phase1_prompt = self._build_reference_phase1_prompt(tags, lyrics, is_negative_prompt=False, negative_prompt=negative_prompt)+ phase1_prompt_negative = self._build_reference_phase1_prompt(tags, lyrics, is_negative_prompt=True, negative_prompt=negative_prompt)+ reserve_metadata = {+ "bpm": _ACE_STEP15_DEFAULT_BPM,+ "duration": max_duration_seconds,+ "keyscale": _ACE_STEP15_DEFAULT_KEYSCALE,+ "timesignature": _ACE_STEP15_DEFAULT_TIMESIGNATURE,+ "caption": tags,+ }+ reserve_cot = self._format_lm_metadata_as_cot(reserve_metadata)+ phase2_prompt = self._build_lm_prompt_with_cot(tags, lyrics, reserve_cot, is_negative_prompt=False, negative_prompt=negative_prompt)+ phase2_prompt_negative = self._build_lm_prompt_with_cot(tags, lyrics, reserve_cot, is_negative_prompt=True, negative_prompt=negative_prompt)++ prompt_len = 0+ for text in (phase1_prompt, phase1_prompt_negative, phase2_prompt, phase2_prompt_negative):+ try:+ prompt_len = max(prompt_len, len(self.lm_tokenizer.encode(text)))+ except Exception:+ pass+ if prompt_len <= 0:+ prompt_len = 256+ reserve_fn(prompt_len=prompt_len, max_tokens=reserve_max_tokens, cfg_scale=reserve_cfg)+def _generate_audio_codes_uncached(self,tags,@@ -1002,6 +1067,17 @@ class ACEStep15Pipeline:abort_fn=self._should_abort,)+ def _release_vllm_lm_runtime(self):+ if self.lm_engine != "vllm" or self._lm_engine_impl is None:+ return+ release_fn = getattr(self._lm_engine_impl, "release_vram", None)+ if not callable(release_fn):+ return+ try:+ release_fn()+ except Exception:+ pass+def _generate_audio_codes(self,tags,@@ -1157,7 +1233,9 @@ class ACEStep15Pipeline:progress_unit="tokens",)if torch.is_tensor(cached):+ self._release_vllm_lm_runtime()return cached.tolist()+ self._release_vllm_lm_runtime()return list(cached)def _default_timbre_latents(self, length):@@ -1795,6 +1873,16 @@ class ACEStep15Pipeline:user_language = Noneif user_language is None and instrumental_only:user_language = "unknown"+ use_ref = "A" in (audio_prompt_type or "")+ has_src_audio = bool(use_ref and audio_guide)+ user_audio_codes = audio_codes if audio_codes is not None else audio_code_hints+ if isinstance(user_audio_codes, str):+ parsed = self._parse_audio_code_string(user_audio_codes)+ user_audio_codes = parsed if parsed else None+ elif isinstance(user_audio_codes, (list, tuple)) and user_audio_codes and isinstance(user_audio_codes[0], str):+ parsed = self._parse_audio_code_string(user_audio_codes[0])+ user_audio_codes = parsed if parsed else None+ keep_vllm_lm_loaded_for_phase2 = bool(self.enable_lm and self.lm_engine == "vllm" and (not has_src_audio) and (not user_audio_codes))phase1_metadata = {"bpm": user_bpm,@@ -1802,6 +1890,11 @@ class ACEStep15Pipeline:"timesignature": user_timesignature,"language": user_language,}+ if self.enable_lm:+ try:+ self._reserve_vllm_runtime_for_max_duration(tags, lyrics, lm_negative_prompt, lm_cfg_scale)+ except Exception:+ passtags_for_generation = tagscomputed_phase1_metadata = {}refined_caption = None@@ -1844,6 +1937,7 @@ class ACEStep15Pipeline:top_p=top_p,top_k=top_k,callback=_phase_callback,+ release_vram_after=not keep_vllm_lm_loaded_for_phase2,)for field_name in missing_fields:value = phase1_metadata.get(field_name, None) if phase1_metadata is not None else None@@ -1900,18 +1994,7 @@ class ACEStep15Pipeline:min_tokens = duration_int * 5meta_cap = self._format_meta(bpm, duration_int, keyscale, timesignature)- use_ref = "A" in (audio_prompt_type or "")use_timbre = "B" in (audio_prompt_type or "")- has_src_audio = bool(use_ref and audio_guide)-- user_audio_codes = audio_codes if audio_codes is not None else audio_code_hints- if isinstance(user_audio_codes, str):- parsed = self._parse_audio_code_string(user_audio_codes)- user_audio_codes = parsed if parsed else None- elif isinstance(user_audio_codes, (list, tuple)) and user_audio_codes:- if isinstance(user_audio_codes[0], str):- parsed = self._parse_audio_code_string(user_audio_codes[0])- user_audio_codes = parsed if parsed else Noneaudio_codes = Noneif user_audio_codes:@@ -2110,13 +2193,3 @@ class ACEStep15Pipeline:"audio_sampling_rate": int(self.audio_sample_rate),"overridden_inputs": overridden_inputs,}-- def close(self):- if self._lm_engine_impl is not None:- close_fn = getattr(self._lm_engine_impl, "close", None)- if callable(close_fn):- try:- close_fn()- except Exception:- pass- self._lm_engine_impl = Nonediff --git a/models/TTS/ace_step15/qwen3_audio_codes.py b/models/TTS/ace_step15/qwen3_audio_codes.pyindex 0ee756a..f1c209b 100644--- a/models/TTS/ace_step15/qwen3_audio_codes.py+++ b/models/TTS/ace_step15/qwen3_audio_codes.py@@ -552,7 +552,9 @@ class Qwen3PtEngine:logits_processor_update_state: Optional[Callable[[int], None]] = None,stop_checker: Optional[Callable[[list[int], int], bool]] = None,progress_label: str = "LM text",+ release_vram_after: bool = True,):+ del release_vram_afterreturn generate_text_legacy(model=self.model,tokenizer=self.tokenizer,@@ -582,13 +584,14 @@ class Qwen3VllmEngine:tokenizer,audio_code_mask: Optional[torch.Tensor],audio_code_token_map: dict[int, int],+ free_mmap_after_cuda_copy: bool = True,weight_load_mode: str = "lazy",):self.lm_weights_path = lm_weights_pathself.tokenizer = tokenizerself.audio_code_mask = audio_code_maskself.audio_code_token_map = audio_code_token_map- self.weight_load_mode = (weight_load_mode or "lazy").lower()+ self.free_mmap_after_cuda_copy = free_mmap_after_cuda_copyself._llm = Noneself._sampling_params_cls = Noneself._model_dir = None@@ -596,6 +599,28 @@ class Qwen3VllmEngine:self._max_num_seqs_hint = Noneself._max_num_batched_tokens_hint = Noneself._last_zero_code_diagnostic = None+ self.weight_load_mode = (weight_load_mode or "lazy").strip().lower()+ if self.weight_load_mode not in ("lazy", "pinned"):+ raise ValueError(f"Unsupported vllm weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")++ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):+ req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(+ prompt_len=prompt_len,+ max_tokens=max_tokens,+ cfg_scale=cfg_scale,+ )+ self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)++ def _release_vram(self):+ if self._llm is None:+ return+ try:+ self._llm.unload_weights()+ except Exception:+ pass++ def release_vram(self):+ self._release_vram()@staticmethoddef _compute_runtime_hints(prompt_len: int, max_tokens: int, cfg_scale: float):@@ -675,22 +700,40 @@ class Qwen3VllmEngine:max_num_batched_tokens=max_num_batched_tokens,gpu_memory_utilization=gpu_memory_utilization,tokenizer=self.tokenizer,+ free_mmap_after_cuda_copy=self.free_mmap_after_cuda_copy,weight_load_mode=self.weight_load_mode,)self._sampling_params_cls = SamplingParamsdef close(self):- if self._llm is None:- return- try:- self._llm.unload_weights()- except Exception:- pass- try:- self._llm.clear_graph_cache()- except Exception:- pass+ llm = getattr(self, "_llm", None)self._llm = None+ if llm is not None:+ try:+ close_fn = getattr(llm, "close", None)+ if callable(close_fn):+ close_fn()+ else:+ try:+ llm.unload_weights()+ except Exception:+ pass+ try:+ llm.clear_graph_cache()+ except Exception:+ pass+ exit_fn = getattr(llm, "exit", None)+ if callable(exit_fn):+ exit_fn()+ except Exception:+ pass+ try:+ del llm+ except Exception:+ pass+ # Keep close() idempotent; this method is also used for runtime rebuilds.+ self._sampling_params_cls = None+ self._model_dir = Nonedef get_last_failure_reason(self) -> str:diag = self._last_zero_code_diagnostic@@ -888,11 +931,7 @@ class Qwen3VllmEngine:progress_unit="tokens",)- if self.weight_load_mode in ("lazy", "pinned"):- try:- self._llm.unload_weights()- except Exception:- pass+ self._release_vram()return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)@@ -912,6 +951,7 @@ class Qwen3VllmEngine:logits_processor_update_state: Optional[Callable[[int], None]] = None,stop_checker: Optional[Callable[[list[int], int], bool]] = None,progress_label: str = "LM text",+ release_vram_after: bool = True,):del stop_checker # nanovllm does not support external stop callbacks yet.if abort_fn is not None and abort_fn():@@ -995,11 +1035,8 @@ class Qwen3VllmEngine:progress_unit="tokens",)- if self.weight_load_mode in ("lazy", "pinned"):- try:- self._llm.unload_weights()- except Exception:- pass+ if release_vram_after:+ self._release_vram()return {"token_ids": token_ids, "text": text}@@ -1012,6 +1049,7 @@ def create_qwen3_lm_engine(lm_weights_path: str,audio_code_mask: Optional[torch.Tensor],audio_code_token_map: dict[int, int],+ free_mmap_after_cuda_copy: bool = True,weight_load_mode: str = "lazy",):engine = (engine_name or "legacy").strip().lower()@@ -1023,6 +1061,7 @@ def create_qwen3_lm_engine(tokenizer=tokenizer,audio_code_mask=audio_code_mask,audio_code_token_map=audio_code_token_map,+ free_mmap_after_cuda_copy=free_mmap_after_cuda_copy,weight_load_mode=weight_load_mode,)if engine == "pt":diff --git a/models/TTS/chatterbox/mtl_tts.py b/models/TTS/chatterbox/mtl_tts.pyindex ac21efc..13047a5 100644--- a/models/TTS/chatterbox/mtl_tts.py+++ b/models/TTS/chatterbox/mtl_tts.py@@ -163,15 +163,15 @@ class ChatterboxMultilingualTTS:ckpt_dir = Path(ckpt_dir)from mmgp import offloadve = VoiceEncoder()- offload.load_model_data(ve, fl.locate_file( os.path.join(ckpt_folder,"ve.safetensors")))+ offload.load_model_data(ve, fl.locate_file(os.path.join(ckpt_folder, "ve.safetensors")), writable_tensors=False)ve.eval()# from accelerate import init_empty_weights# with init_empty_weights():t3 = T3(T3Config.multilingual())- offload.load_model_data(t3, fl.locate_file( os.path.join(ckpt_folder,"t3_mtl23ls_v2.safetensors")))+ offload.load_model_data(t3, fl.locate_file(os.path.join(ckpt_folder, "t3_mtl23ls_v2.safetensors")), writable_tensors=False)t3.eval()s3gen = S3Gen()- offload.load_model_data(s3gen, fl.locate_file( os.path.join(ckpt_folder,"s3gen.pt")))+ offload.load_model_data(s3gen, fl.locate_file(os.path.join(ckpt_folder, "s3gen.pt")), writable_tensors=False)s3gen.eval()tokenizer = MTLTokenizer(diff --git a/models/TTS/yue/pipeline.py b/models/TTS/yue/pipeline.pyindex 554da84..68fd4bf 100644--- a/models/TTS/yue/pipeline.py+++ b/models/TTS/yue/pipeline.py@@ -150,6 +150,7 @@ class YuePipeline:stage1_weights_path,modelClass=LlamaForCausalLM,defaultConfigPath=stage1_config_path,+ writable_tensors=False,configKwargs={"attn_implementation": attn_implementation},default_dtype=torch.bfloat16,ignore_unused_weights=True,@@ -164,6 +165,7 @@ class YuePipeline:stage2_weights_path,modelClass=LlamaForCausalLM,defaultConfigPath=stage2_config_path,+ writable_tensors=False,configKwargs={"attn_implementation": attn_implementation},default_dtype=torch.float16,ignore_unused_weights=True,diff --git a/models/flux/flux_main.py b/models/flux/flux_main.pyindex 062bfc9..60b6db3 100644--- a/models/flux/flux_main.py+++ b/models/flux/flux_main.py@@ -151,6 +151,7 @@ class model_factory:fl.locate_file(os.path.join("siglip-so400m-patch14-384", "model.safetensors")),modelClass=SiglipVisionModel,defaultConfigPath=fl.locate_file(os.path.join("siglip-so400m-patch14-384", "vision_config.json")),+ writable_tensors=False,)siglip_model.eval().to("cpu")if len(model_filename) > 1:@@ -162,7 +163,7 @@ class model_factory:hidden_size=3072, #self.hidden_size,context_layer_norm=True,)- offload.load_model_data(feature_embedder, model_filename[1])+ offload.load_model_data(feature_embedder, model_filename[1], writable_tensors=False)self.vision_encoder = siglip_modelself.vision_encoder_processor = siglip_processorself.feature_embedder = feature_embedderdiff --git a/models/flux/modules/conditioner.py b/models/flux/modules/conditioner.pyindex c37d09b..afc1d8f 100644--- a/models/flux/modules/conditioner.py+++ b/models/flux/modules/conditioner.py@@ -12,13 +12,13 @@ class HFEmbedder(nn.Module):if is_clip:from mmgp import offloadself.tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(version, max_length=max_length)- self.hf_module= offload.fast_load_transformers_model(os.path.join(version, "model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = os.path.join(version, "text_config.json"))+ self.hf_module= offload.fast_load_transformers_model(os.path.join(version, "model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = os.path.join(version, "text_config.json"), writable_tensors=False)# self.model.final_layer_norm = self.model.text_model.final_layer_norm# self.hf_module: CLIPTextModel = CLIPTextModel.from_pretrained(version, **hf_kwargs)else:from mmgp import offload as offloadobjself.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(os.path.dirname(text_encoder_filename), max_length=max_length)- self.hf_module: T5EncoderModel = offloadobj.fast_load_transformers_model(text_encoder_filename)+ self.hf_module: T5EncoderModel = offloadobj.fast_load_transformers_model(text_encoder_filename, writable_tensors=False)self.hf_module = self.hf_module.eval().requires_grad_(False)diff --git a/models/flux/util.py b/models/flux/util.pyindex ea28895..1e08915 100644--- a/models/flux/util.py+++ b/models/flux/util.py@@ -1017,7 +1017,7 @@ def load_flow_model(# print(f"Loading checkpoint: {ckpt_path}")from mmgp import offload- offload.load_model_data(model, model_filename, preprocess_sd=preprocess_sd)+ offload.load_model_data(model, model_filename, preprocess_sd=preprocess_sd, writable_tensors=False)# # load_sft doesn't support torch.device# sd = load_sft(ckpt_path, device=str(device))diff --git a/models/hyvideo/hunyuan.py b/models/hyvideo/hunyuan.pyindex 2b49982..c6d26a3 100644--- a/models/hyvideo/hunyuan.py+++ b/models/hyvideo/hunyuan.py@@ -259,7 +259,7 @@ class Inference(object):from mmgp import offloadsource = model_def.get("source", None)- kwargs_light = dict(do_quantize= quantizeTransformer and not save_quantized, pinToMemory = pinToMemory, partialPinning = partialPinning)+ kwargs_light = dict(do_quantize= quantizeTransformer and not save_quantized, pinToMemory = pinToMemory, partialPinning = partialPinning, writable_tensors=False)if source is not None:from wgp import save_modeloffload.load_model_data(model, fl.locate_file(source), **kwargs_light)@@ -338,7 +338,7 @@ class Inference(object):from accelerate import init_empty_weightswith init_empty_weights():vae = AutoencoderKLConv3D(**config)- offload.load_model_data(vae, vae_filepath )+ offload.load_model_data(vae, vae_filepath, writable_tensors=False)vae = vae.to("cpu")s_ratio = t_ratio = 1vae._model_dtype = torch.float32 if VAE_dtype == torch.float32 else torch.float16diff --git a/models/hyvideo/text_encoder/__init__.py b/models/hyvideo/text_encoder/__init__.pyindex c770a41..9719331 100644--- a/models/hyvideo/text_encoder/__init__.py+++ b/models/hyvideo/text_encoder/__init__.py@@ -192,13 +192,13 @@ class TextEncoder(nn.Module):from mmgp import offloadif "llm" in text_encoder_type:if "i2v" in text_encoder_type:- self.model= offload.fast_load_transformers_model(self.model_path, modelClass= LlavaForConditionalGeneration)+ self.model= offload.fast_load_transformers_model(self.model_path, modelClass= LlavaForConditionalGeneration, writable_tensors=False)else:- self.model= offload.fast_load_transformers_model(self.model_path, modelPrefix="language_model", forcedConfigPath = fl.locate_file("llava-llama-3-8b/config.json"))+ self.model= offload.fast_load_transformers_model(self.model_path, modelPrefix="language_model", forcedConfigPath = fl.locate_file("llava-llama-3-8b/config.json"), writable_tensors=False)self.model.final_layer_norm = self.model.model.normelse:- self.model= offload.fast_load_transformers_model(fl.locate_file("clip_vit_large_patch14/model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = fl.locate_file("clip_vit_large_patch14/text_config.json"))+ self.model= offload.fast_load_transformers_model(fl.locate_file("clip_vit_large_patch14/model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = fl.locate_file("clip_vit_large_patch14/text_config.json"), writable_tensors=False)self.model.final_layer_norm = self.model.text_model.final_layer_normself.dtype = self.model.dtypediff --git a/models/hyvideo/vae/__init__.py b/models/hyvideo/vae/__init__.pyindex 73c0032..6d2b422 100644--- a/models/hyvideo/vae/__init__.py+++ b/models/hyvideo/vae/__init__.py@@ -54,7 +54,7 @@ def load_vae(vae_type: str="884-16c-hy",# offload.save_model(vae, "vae_32.safetensors")# vae.to(torch.bfloat16)# offload.save_model(vae, "vae_16.safetensors")- offload.load_model_data(vae, vae_path )+ offload.load_model_data(vae, vae_path, writable_tensors=False)# ckpt = torch.load(vae_ckpt, weights_only=True, map_location=vae.device)spatial_compression_ratio = vae.config.spatial_compression_ratiodiff --git a/models/kandinsky5/kandinsky/models/text_embedders.py b/models/kandinsky5/kandinsky/models/text_embedders.pyindex 42d183e..075fff6 100644--- a/models/kandinsky5/kandinsky/models/text_embedders.py+++ b/models/kandinsky5/kandinsky/models/text_embedders.py@@ -16,7 +16,7 @@ from shared.utils.text_encoder_cache import TextEncoderCacheclass ClipTextEmbedder:def __init__(self, conf, device):- self.model= offload.fast_load_transformers_model( os.path.join(conf.checkpoint_path,"model.safetensors"), modelClass=CLIPTextModel, ignore_unused_weights= True, forcedConfigPath = os.path.join(conf.checkpoint_path, "text_config.json"))+ self.model= offload.fast_load_transformers_model(os.path.join(conf.checkpoint_path, "model.safetensors"), modelClass=CLIPTextModel, ignore_unused_weights= True, forcedConfigPath=os.path.join(conf.checkpoint_path, "text_config.json"), writable_tensors=False)self.model.final_layer_norm = self.model.text_model.final_layer_normself.model.eval()# self.model = freeze(self.model)diff --git a/models/longcat/longcat_main.py b/models/longcat/longcat_main.pyindex cd2c762..aa32c13 100644--- a/models/longcat/longcat_main.py+++ b/models/longcat/longcat_main.py@@ -147,6 +147,7 @@ class LongCatModel:vae_weights,modelClass=AutoencoderKLWan,defaultConfigPath=fl.locate_file(vae_cfg_path),+ writable_tensors=False,preprocess_sd=preprocess_vae_sd,default_dtype=VAE_dtype,)diff --git a/models/ltx2/ltx2.py b/models/ltx2/ltx2.pyindex 93a83ee..cc116d1 100644--- a/models/ltx2/ltx2.py+++ b/models/ltx2/ltx2.py@@ -499,6 +499,7 @@ class LTX2:path,postprocess_sd=postprocess,default_dtype=self.dtype,+ writable_tensors=False,ignore_missing_keys=False,)model.eval().requires_grad_(False)diff --git a/models/ltx2/ltx_core/loader/single_gpu_model_builder.py b/models/ltx2/ltx_core/loader/single_gpu_model_builder.pyindex aa2d136..c276342 100644--- a/models/ltx2/ltx_core/loader/single_gpu_model_builder.py+++ b/models/ltx2/ltx_core/loader/single_gpu_model_builder.py@@ -141,6 +141,7 @@ class SingleGPUModelBuilder(Generic[ModelType], ModelBuilderProtocol[ModelType],meta_model,[(sd, quantization_map)],default_dtype=dtype or torch.bfloat16,+ writable_tensors=False,ignore_missing_keys=self.ignore_missing_keys,)return self._return_model(meta_model, device)@@ -163,6 +164,7 @@ class SingleGPUModelBuilder(Generic[ModelType], ModelBuilderProtocol[ModelType],meta_model,model_paths,default_dtype=dtype or torch.bfloat16,+ writable_tensors=False,preprocess_sd=preprocess_sd,ignore_missing_keys=self.ignore_missing_keys,)diff --git a/models/ltx_video/ltxv.py b/models/ltx_video/ltxv.pyindex 8038987..5898576 100644--- a/models/ltx_video/ltxv.py+++ b/models/ltx_video/ltxv.py@@ -178,7 +178,7 @@ class LTXV:# offload.save_model(transformer, "ckpts/ltxv_0.9.8_13B_quanto_bf16_int8.safetensors", do_quantize= True, config_file_path= "c:/temp/ltxv_config.json")# vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)- vae = offload.fast_load_transformers_model( fl.locate_file("ltxv_0.9.7_VAE.safetensors"), modelClass=CausalVideoAutoencoder)+ vae = offload.fast_load_transformers_model(fl.locate_file("ltxv_0.9.7_VAE.safetensors"), modelClass=CausalVideoAutoencoder, writable_tensors=False)# vae = offload.fast_load_transformers_model("ckpts/ltxv_0.9.8_VAE.safetensors", modelClass=CausalVideoAutoencoder)# if VAE_dtype == torch.float16:VAE_dtype = torch.bfloat16@@ -188,7 +188,7 @@ class LTXV:# offload.save_model(vae, "vae.safetensors", config_file_path="c:/temp/config_vae.json")# model_filepath = "c:/temp/ltxd/ltxv-13b-0.9.7-distilled.safetensors"- transformer = offload.fast_load_transformers_model(model_filepath, modelClass=Transformer3DModel)+ transformer = offload.fast_load_transformers_model(model_filepath, modelClass=Transformer3DModel, writable_tensors=False)# offload.save_model(transformer, "ckpts/ltxv_0.9.7_13B_distilled_bf16.safetensors", config_file_path= "c:/temp/ltxd/config.json")# offload.save_model(transformer, "ckpts/ltxv_0.9.7_13B_distilled_quanto_bf16_int8.safetensors", do_quantize= True, config_file_path="c:/temp/ltxd/config.json")# transformer = offload.fast_load_transformers_model(model_filepath, modelClass=Transformer3DModel)@@ -215,7 +215,7 @@ class LTXV:# offload.save_model(text_encoder, "T5_xxl_1.1_enc_bf16.safetensors", config_file_path="T5_config.json")# offload.save_model(text_encoder, "T5_xxl_1.1_enc_quanto_bf16_int8.safetensors", do_quantize= True, config_file_path="T5_config.json")- text_encoder = offload.fast_load_transformers_model(text_encoder_filepath)+ text_encoder = offload.fast_load_transformers_model(text_encoder_filepath, writable_tensors=False)patchifier = SymmetricPatchifier(patch_size=1)text_encoder_folder = self.model_def.get("text_encoder_folder")if text_encoder_folder:diff --git a/models/qwen/qwen_main.py b/models/qwen/qwen_main.pyindex 808d225..e9c6498 100644--- a/models/qwen/qwen_main.py+++ b/models/qwen/qwen_main.py@@ -87,9 +87,9 @@ class model_factory():source = model_def.get("source", None)if source is not None:- offload.load_model_data(transformer, source, fused_split_map=_QWEN_FUSED_SPLIT_MAP)+ offload.load_model_data(transformer, source, fused_split_map=_QWEN_FUSED_SPLIT_MAP, writable_tensors=False)else:- offload.load_model_data(transformer, transformer_filename, fused_split_map=_QWEN_FUSED_SPLIT_MAP)+ offload.load_model_data(transformer, transformer_filename, fused_split_map=_QWEN_FUSED_SPLIT_MAP, writable_tensors=False)# transformer = offload.fast_load_transformers_model("transformer_quanto.safetensors", writable_tensors= True , modelClass=QwenImageTransformer2DModel, defaultConfigPath="transformer_config.json")if not source is None:diff --git a/models/wan/any2video.py b/models/wan/any2video.pyindex df985d1..e886008 100644--- a/models/wan/any2video.py+++ b/models/wan/any2video.py@@ -953,7 +953,7 @@ class WanAny2V:if True:with init_empty_weights():arc_resampler = Resampler( depth=4, dim=1280, dim_head=64, embedding_dim=512, ff_mult=4, heads=20, num_queries=16, output_dim=2048 if lynx_lite else 5120 )- offload.load_model_data(arc_resampler, fl.locate_file("wan2.1_lynx_lite_arc_resampler.safetensors" if lynx_lite else "wan2.1_lynx_full_arc_resampler.safetensors"))+ offload.load_model_data(arc_resampler, fl.locate_file("wan2.1_lynx_lite_arc_resampler.safetensors" if lynx_lite else "wan2.1_lynx_full_arc_resampler.safetensors"), writable_tensors=False)arc_resampler.to(self.device)arcface_embed = face_arc_embeds[None,None,:].to(device=self.device, dtype=torch.float)ip_hidden_states = arc_resampler(arcface_embed).to(self.dtype)diff --git a/models/wan/fantasytalking/infer.py b/models/wan/fantasytalking/infer.pyindex a756417..8bb0917 100644--- a/models/wan/fantasytalking/infer.py+++ b/models/wan/fantasytalking/infer.py@@ -17,7 +17,7 @@ def parse_audio(audio_path, start_frame, num_frames, fps = 23, device = "cuda"):with init_empty_weights():proj_model = AudioProjModel( 768, 2048)- offload.load_model_data(proj_model, fl.locate_file("fantasy_proj_model.safetensors"))+ offload.load_model_data(proj_model, fl.locate_file("fantasy_proj_model.safetensors"), writable_tensors=False)proj_model.to("cpu").eval().requires_grad_(False)wav2vec_model_dir = fl.locate_folder("wav2vec")@@ -34,4 +34,4 @@ def parse_audio(audio_path, start_frame, num_frames, fps = 23, device = "cuda"):gc.collect()torch.cuda.empty_cache()- return audio_proj_split, audio_context_lens\ No newline at end of file+ return audio_proj_split, audio_context_lensdiff --git a/models/wan/multitalk/multitalk.py b/models/wan/multitalk/multitalk.pyindex ca732c3..f0a65ce 100644--- a/models/wan/multitalk/multitalk.py+++ b/models/wan/multitalk/multitalk.py@@ -28,6 +28,7 @@ def custom_init(device, wav2vec):os.path.join(wav2vec_folder, "pytorch_model.bin"),modelClass=Wav2Vec2Model,defaultConfigPath=os.path.join(wav2vec_folder, "config.json"),+ writable_tensors=False,).to(device)audio_encoder.feature_extractor._freeze_parameters()wav2vec_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(diff --git a/models/wan/ovi_fusion_engine.py b/models/wan/ovi_fusion_engine.pyindex 0474231..437c4f1 100644--- a/models/wan/ovi_fusion_engine.py+++ b/models/wan/ovi_fusion_engine.py@@ -71,8 +71,8 @@ class OviFusionEngine:self.target_dtype = torch.bfloat16 # dtype, wont work with torch.float16model, video_config, audio_config = init_fusion_score_model_ovi()# offload.load_model_data(model, "c:/temp/model_960x960.safetensors")- offload.load_model_data(model.video_model, model_filename[0])- offload.load_model_data(model.audio_model, model_filename[1])+ offload.load_model_data(model.video_model, model_filename[0], writable_tensors=False)+ offload.load_model_data(model.audio_model, model_filename[1], writable_tensors=False)offload.change_dtype(model, dtype, True)model = model.eval()# model.set_rope_params()diff --git a/preprocessing/matanyone/tools/base_segmenter.py b/preprocessing/matanyone/tools/base_segmenter.pyindex 09fa309..5327037 100644--- a/preprocessing/matanyone/tools/base_segmenter.py+++ b/preprocessing/matanyone/tools/base_segmenter.py@@ -139,7 +139,7 @@ class BaseSegmenter:# self.model.to(torch.float16)# offload.save_model(self.model, "ckpts/mask/sam_vit_h_4b8939_fp16.safetensors")- offload.load_model_data(self.model, fl.locate_file("mask/sam_vit_h_4b8939_fp16.safetensors"))+ offload.load_model_data(self.model, fl.locate_file("mask/sam_vit_h_4b8939_fp16.safetensors"), writable_tensors=False)self.model.to(torch.float32) # need to be optimized, if not f32 crappy precisionself.model.to(device=self.device)self.predictor = SamPredictor(self.model)diff --git a/shared/llm_engines/nanovllm/config.py b/shared/llm_engines/nanovllm/config.pyindex 69513e2..c39c3ea 100644--- a/shared/llm_engines/nanovllm/config.py+++ b/shared/llm_engines/nanovllm/config.py@@ -12,8 +12,9 @@ class Config:gpu_memory_utilization: float = 0.9tensor_parallel_size: int = 1enforce_eager: bool = False- weight_load_mode: str = "eager" # eager | lazy | pinned+ weight_load_mode: str = "lazy" # lazy | pinnedhf_config: AutoConfig | None = None+ free_mmap_after_cuda_copy: bool = Trueeos: int = -1kvcache_block_size: int = 256num_kvcache_blocks: int = -1@@ -29,6 +30,8 @@ class Config:self.model_dir = self.modelassert self.kvcache_block_size % 256 == 0assert 1 <= self.tensor_parallel_size <= 8+ if self.weight_load_mode not in ("lazy", "pinned"):+ raise ValueError(f"Unsupported weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")self.hf_config = AutoConfig.from_pretrained(self.model_dir)self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings)assert self.max_num_batched_tokens >= self.max_model_lendiff --git a/shared/llm_engines/nanovllm/engine/llm_engine.py b/shared/llm_engines/nanovllm/engine/llm_engine.pyindex b48d179..6edec65 100644--- a/shared/llm_engines/nanovllm/engine/llm_engine.py+++ b/shared/llm_engines/nanovllm/engine/llm_engine.py@@ -4,7 +4,6 @@ from time import perf_counterfrom tqdm.auto import tqdmfrom transformers import AutoTokenizerimport torch.multiprocessing as mp-import torchfrom nanovllm.config import Configfrom nanovllm.sampling_params import SamplingParams@@ -38,22 +37,80 @@ class LLMEngine:self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True)config.eos = self.tokenizer.eos_token_idself.scheduler = Scheduler(config)- atexit.register(self.exit)+ self._exit_registered = False+ self._closed = False+ self._exited = False+ self._atexit_callback = self.exit+ atexit.register(self._atexit_callback)+ self._exit_registered = Truedef exit(self):- self.model_runner.call("exit")- del self.model_runner- for p in self.ps:- p.join()+ if self._exited:+ return+ self._exited = True+ runner = getattr(self, "model_runner", None)+ if runner is not None:+ try:+ runner.call("exit")+ except Exception:+ pass+ try:+ del self.model_runner+ except Exception:+ pass+ for p in list(getattr(self, "ps", [])):+ try:+ p.join()+ except Exception:+ pass+ self.ps = []+ self.events = []++ def close(self):+ if self._closed:+ return+ self._closed = True+ try:+ self.unload_weights()+ except Exception:+ pass+ try:+ self.clear_graph_cache()+ except Exception:+ pass+ try:+ self.exit()+ except Exception:+ pass+ if self._exit_registered:+ try:+ atexit.unregister(self._atexit_callback)+ except Exception:+ pass+ self._exit_registered = False+ self._atexit_callback = None+ self.scheduler = None+ self.tokenizer = None++ def __del__(self):+ try:+ self.close()+ except Exception:+ passdef unload_weights(self):- self.model_runner.unload_weights()+ runner = getattr(self, "model_runner", None)+ if runner is None:+ return+ runner.unload_weights()# KV cache is invalid after unload/reload, so cached prefix block metadata# must be dropped as well to prevent stale-cache reuse.try:self.reset()except Exception:pass+ if self.scheduler is None:+ returnself.scheduler.waiting.clear()self.scheduler.running.clear()self.scheduler.block_manager = BlockManager(@@ -62,13 +119,22 @@ class LLMEngine:)def clear_graph_cache(self):- self.model_runner.clear_graph_cache()+ runner = getattr(self, "model_runner", None)+ if runner is None:+ return+ runner.clear_graph_cache()def reset_guard_counts(self):- self.model_runner.call("reset_guard_counts")+ runner = getattr(self, "model_runner", None)+ if runner is None:+ return+ runner.call("reset_guard_counts")def get_guard_counts(self, reset: bool = False):- return self.model_runner.call("get_guard_counts", reset)+ runner = getattr(self, "model_runner", None)+ if runner is None:+ return {}+ return runner.call("get_guard_counts", reset)def add_request(self, prompt: str | list[int], sampling_params: SamplingParams, unconditional_prompt: str | list[int] | None = None):if isinstance(prompt, str):@@ -109,6 +175,8 @@ class LLMEngine:return outputs, num_tokensdef is_finished(self):+ if self.scheduler is None:+ return Truereturn self.scheduler.is_finished()def reset(self):@@ -118,6 +186,8 @@ class LLMEngine:KV cache block leaks that can cause 'deque index out of range' errors."""# Deallocate all running sequences+ if self.scheduler is None:+ returnwhile self.scheduler.running:seq = self.scheduler.running.popleft()if seq.block_table: # Only deallocate if blocks are allocated@@ -136,6 +206,8 @@ class LLMEngine:use_tqdm: bool = True,unconditional_prompts: list[str] | list[list[int]] | None = None,) -> list[str]:+ if self.scheduler is None:+ raise RuntimeError("LLM engine is closed.")# Ensure weights/KV cache are ready for lazy/pinned modes, and sync scheduler blocks.self.model_runner.ensure_weights_loaded()if (self.config.num_kvcache_blocks > 0 and@@ -163,10 +235,7 @@ class LLMEngine:breakexcept Exception:seed_to_apply = None- if seed_to_apply is not None:- torch.manual_seed(seed_to_apply)- if torch.cuda.is_available():- torch.cuda.manual_seed(seed_to_apply)+ self.model_runner.call("set_sampling_seed", seed_to_apply)if unconditional_prompts is None:unconditional_prompts = [None] * len(prompts)for prompt, sp, uncond_prompt in zip(prompts, sampling_params, unconditional_prompts):diff --git a/shared/llm_engines/nanovllm/engine/model_runner.py b/shared/llm_engines/nanovllm/engine/model_runner.pyindex f9015a5..9b765ee 100644--- a/shared/llm_engines/nanovllm/engine/model_runner.py+++ b/shared/llm_engines/nanovllm/engine/model_runner.py@@ -1,4 +1,5 @@import pickle+import gcimport torchimport torch.distributed as distfrom multiprocessing.synchronize import Event@@ -7,7 +8,7 @@ import sysfrom nanovllm.config import Configfrom nanovllm.engine.sequence import Sequence-from nanovllm.models.qwen3 import Qwen3ForCausalLM+from nanovllm.models import resolve_model_classfrom nanovllm.layers.sampler import Samplerfrom nanovllm.utils.context import set_context, get_context, reset_contextfrom nanovllm.utils.loader import load_model, WeightStore@@ -90,39 +91,26 @@ class ModelRunner:config_dtype = torch.bfloat16self.dtype = config_dtype # Save for later use- self.weight_load_mode = (config.weight_load_mode or "eager").lower()+ self.weight_load_mode = (config.weight_load_mode or "lazy").lower()self._weights_loaded = Falseself._weight_store = Noneself._is_quanto_int8 = False+ self._model_cls = resolve_model_class(hf_config)+ self._model_is_meta = Falseself._graph_cache = {}self._graph_cache_order = []self._logits_bias_cache = {}+ self._sampling_generator = Noneself._guard_counts = {}self._guard_seen_details = set()torch.set_default_dtype(config_dtype)- if self.weight_load_mode in ("lazy", "pinned"):- torch.set_default_device("cpu")- self.model = Qwen3ForCausalLM(hf_config)- self._weight_store = WeightStore(config.model_file or config.model_dir, mode=self.weight_load_mode)- self._is_quanto_int8 = bool(getattr(self._weight_store, "is_quanto_int8", False))- else:- torch.set_default_device("cuda")- self.model = Qwen3ForCausalLM(hf_config)- load_model(self.model, config.model_file or config.model_dir)- self._retie_word_embeddings_if_needed()- self._weights_loaded = True+ self.model = self._build_deferred_model(hf_config)self.sampler = Sampler()# Pre-allocate buffers for sampling (optimization: avoid repeated tensor creation)- # Must be called before warmup_model() since it uses these buffers+ # Must be called before model execution paths that use these buffers.self._allocate_sample_buffers()- if self._weights_loaded:- self.warmup_model()- self.allocate_kv_cache()- if not self.enforce_eager:- self.capture_cudagraph()-torch.set_default_device("cpu")torch.set_default_dtype(default_dtype)@@ -135,59 +123,125 @@ class ModelRunner:self.shm = SharedMemory(name="nanovllm")self.loop()+ def _build_deferred_model(self, hf_config):+ # Lazy/pinned mode should not allocate full CPU parameter storage upfront.+ with torch.device("meta"):+ model = self._model_cls(hf_config)+ self._model_is_meta = True+ return model+def ensure_weights_loaded(self):if self._weights_loaded:return+ print(f"Loading LM Weights '{self.config.model}' in vllm...")default_dtype = torch.get_default_dtype()torch.set_default_dtype(self.dtype)- torch.set_default_device("cuda")- if self._is_quanto_int8:- for module in self.model.modules():- prepare = getattr(module, "prepare_for_quantized_load", None)- if callable(prepare):- prepare()- self.model = self.model.to("cuda")- load_model(self.model, "", weight_store=self._weight_store)- self._retie_word_embeddings_if_needed()- self._weights_loaded = True- self.warmup_model()- self.allocate_kv_cache()- if not self.enforce_eager:- self.capture_cudagraph()- torch.set_default_device("cpu")- torch.set_default_dtype(default_dtype)-- def _retie_word_embeddings_if_needed(self):- # Some quantized checkpoints omit lm_head.weight and rely on tied embeddings.- # After device moves/load cycles, the tie can be broken; restore it explicitly.try:- lm_head = getattr(self.model, "lm_head", None)- embed = getattr(self.model, "embed_tokens", None)- if lm_head is None or embed is None:- return- lm_w = getattr(lm_head, "weight", None)- emb_w = getattr(embed, "weight", None)- if lm_w is None or emb_w is None:- return- if lm_w.shape != emb_w.shape:- return- if lm_w.data_ptr() != emb_w.data_ptr():- lm_head.weight.data = emb_w.data- except Exception:+ self._ensure_weight_store()+ if self.model is None:+ self.model = self._build_deferred_model(self.config.hf_config)+ self._tie_word_embeddings_if_needed()+ self._mark_tied_meta_parameter_ids()+ # Tie before materialization so lm_head does not allocate a duplicate tensor.+ was_meta_model = bool(self._model_is_meta)+ if self._is_quanto_int8:+ for module in self.model.modules():+ prepare = getattr(module, "prepare_for_quantized_load", None)+ if callable(prepare):+ prepare()+ # Keep model meta until each parameter is replaced directly from safetensor+ # to avoid allocating a dense, fully materialized CUDA shell first.+ load_model(+ self.model,+ "",+ weight_store=self._weight_store,+ # We keep tensors tied to the active weight reader only within each+ # load iteration; explicit cloning is not needed and avoids a full+ # model-sized temporary copy in RAM.+ clone_loaded_tensors=False,+ )+ if was_meta_model:+ self._materialize_runtime_buffers()+ self._model_is_meta = False+ self._close_weight_store_if_lazy()+ self._weights_loaded = True+ self.allocate_kv_cache()+ if not self.enforce_eager:+ self.capture_cudagraph()+ finally:+ if not self._weights_loaded:+ self._close_weight_store_if_lazy()+ torch.set_default_device("cpu")+ torch.set_default_dtype(default_dtype)++ def _ensure_weight_store(self):+ if self._weight_store is not None:+ return+ self._weight_store = WeightStore(self.config.model_file or self.config.model_dir, mode=self.weight_load_mode)+ self._is_quanto_int8 = bool(getattr(self._weight_store, "is_quanto_int8", False))++ def _close_weight_store_if_lazy(self):+ if self._weight_store is None:+ return+ close_fn = getattr(self._weight_store, "close", None)+ if callable(close_fn):+ close_fn()+ self._weight_store = None++ def _materialize_runtime_buffers(self):+ if self.model is None:+ return+ for module in self.model.modules():+ materialize = getattr(module, "materialize_cache", None)+ if callable(materialize):+ materialize()++ def _get_tied_embeddings(self):+ if self.model is None:+ return None+ hf_config = getattr(self.config, "hf_config", None)+ if not bool(getattr(hf_config, "tie_word_embeddings", False)):+ return None+ lm_head = getattr(self.model, "lm_head", None)+ embed = getattr(self.model, "embed_tokens", None)+ if lm_head is None or embed is None:+ return None+ lm_w = getattr(lm_head, "weight", None)+ emb_w = getattr(embed, "weight", None)+ if lm_w is None or emb_w is None:+ return None+ return lm_head, embed++ def _mark_tied_meta_parameter_ids(self):+ tied = self._get_tied_embeddings()+ if tied is None:+ self.model._vllm_tied_param_ids = set()+ return+ lm_head, embed = tied+ if lm_head.weight is not embed.weight:+ self.model._vllm_tied_param_ids = set()+ return+ self.model._vllm_tied_param_ids = {id(embed.weight)}++ def _tie_word_embeddings_if_needed(self):+ tied = self._get_tied_embeddings()+ if tied is None:return+ lm_head, embed = tied+ if lm_head.weight is not embed.weight:+ lm_head.register_parameter("weight", embed.weight)+ if lm_head.weight is not embed.weight:+ raise RuntimeError("Failed to retie lm_head.weight with embed_tokens.weight.")def unload_weights(self):if not self._weights_loaded:return- try:- self.model = self.model.to("cpu")- except Exception:- pass# Clear attention KV cache refs so we don't write into freed storage later.try:- for module in self.model.modules():- if hasattr(module, "k_cache") and hasattr(module, "v_cache"):- module.k_cache = module.v_cache = torch.tensor([])+ if self.model is not None:+ for module in self.model.modules():+ if hasattr(module, "k_cache") and hasattr(module, "v_cache"):+ module.k_cache = module.v_cache = torch.tensor([])except Exception:passif hasattr(self, "kv_cache"):@@ -213,7 +267,19 @@ class ModelRunner:except Exception:passself._logits_bias_cache.clear()+ self._sampling_generator = Noneself._weights_loaded = False+ self.model = None+ self._model_is_meta = False+ if self._weight_store is not None:+ try:+ close_fn = getattr(self._weight_store, "close", None)+ if callable(close_fn):+ close_fn()+ except Exception:+ pass+ self._weight_store = None+ gc.collect()def _get_graph_capture_signature(self):model_ptr = -1@@ -231,6 +297,12 @@ class ModelRunner:passreturn (model_ptr, kv_ptr, int(self.config.max_model_len), int(self.config.max_num_seqs))+ def _get_model_device(self) -> torch.device:+ try:+ return next(self.model.parameters()).device+ except Exception:+ return torch.device("cpu")+def _drop_graph_cache_entry(self, cache_key):entry = self._graph_cache.pop(cache_key, None)if cache_key in self._graph_cache_order:@@ -286,6 +358,14 @@ class ModelRunner:self._logits_bias_cache[key] = cachedreturn cached+ def set_sampling_seed(self, seed: int | None):+ if seed is None or not torch.cuda.is_available():+ self._sampling_generator = None+ return+ generator = torch.Generator(device="cuda")+ generator.manual_seed(int(seed))+ self._sampling_generator = generator+@staticmethoddef _apply_logits_bias(logits_row: torch.Tensor, bias: torch.Tensor):logits_row.add_(bias)@@ -323,7 +403,53 @@ class ModelRunner:# Max length is max_model_len since sequences can be that longself._seq_token_ids_buffer = torch.zeros(max_bs, self.config.max_model_len, dtype=torch.int64, device="cpu", pin_memory=True)+ def _release_sample_buffers(self):+ buffer_names = [+ "_cpu_temperatures",+ "_cpu_cfg_scales",+ "_cpu_top_ks",+ "_cpu_top_ps",+ "_cpu_repetition_penalties",+ "_cpu_input_ids",+ "_cpu_positions",+ "_cpu_slot_mapping",+ "_cpu_context_lens",+ "_cpu_prefill_input_ids",+ "_cpu_prefill_positions",+ "_cpu_prefill_cu_seqlens",+ "_cpu_prefill_slot_mapping",+ "_cpu_block_tables",+ "_seq_token_ids_buffer",+ ]+ for name in buffer_names:+ if hasattr(self, name):+ try:+ delattr(self, name)+ except Exception:+ pass+def exit(self):+ try:+ self.unload_weights()+ except Exception:+ pass+ try:+ if self._weight_store is not None:+ close_fn = getattr(self._weight_store, "close", None)+ if callable(close_fn):+ close_fn()+ except Exception:+ pass+ self._weight_store = None+ self._release_sample_buffers()+ self._logits_bias_cache.clear()+ self._guard_counts.clear()+ self._guard_seen_details.clear()+ if hasattr(self, "sampler"):+ self.sampler = None+ if hasattr(self, "model"):+ self.model = None+ self._model_is_meta = Falseif self.world_size > 1:self.shm.close()dist.barrier()@@ -338,6 +464,10 @@ class ModelRunner:torch.cuda.synchronize()except Exception:pass+ try:+ torch.cuda.empty_cache()+ except Exception:+ passif dist.is_initialized():dist.destroy_process_group()@@ -371,58 +501,51 @@ class ModelRunner:method = getattr(self, method_name, None)return method(*args)- def warmup_model(self):- torch.cuda.empty_cache()- torch.cuda.reset_peak_memory_stats()- max_num_batched_tokens, max_model_len = self.config.max_num_batched_tokens, self.config.max_model_len- num_seqs = min(max_num_batched_tokens // max_model_len, self.config.max_num_seqs)- seqs = [Sequence([0] * max_model_len) for _ in range(num_seqs)]- self.run(seqs, True)- torch.cuda.empty_cache()-def allocate_kv_cache(self):config = self.confighf_config = config.hf_config- free, total = torch.cuda.mem_get_info()- current = torch.cuda.memory_stats()["allocated_bytes.all.current"]num_kv_heads = hf_config.num_key_value_heads // self.world_sizehead_dim = getattr(hf_config, "head_dim", hf_config.hidden_size // hf_config.num_attention_heads)block_bytes = 2 * hf_config.num_hidden_layers * self.block_size * num_kv_heads * head_dim * self.dtype.itemsize-- # Calculate available memory for KV cache- # After warmup_model, empty_cache has been called, so current represents model memory only- # Use free memory but respect the gpu_memory_utilization limit- target_total_usage = total * config.gpu_memory_utilization- available_for_kv_cache = min(free * 0.9, target_total_usage - current)-- # Ensure we have positive memory available- if available_for_kv_cache <= 0:- available_for_kv_cache = free * 0.5 # Fallback to 50% of free memory-- config.num_kvcache_blocks = max(1, int(available_for_kv_cache) // block_bytes)- # Cap KV cache blocks to what is required by max_model_len and max_num_seqs.- # This keeps VRAM usage proportional to the requested token budget (incl. CFG).++ # Strict policy: allocate exactly the blocks required by requested runtime limits.required_blocks_per_seq = (config.max_model_len + self.block_size - 1) // self.block_sizerequired_total_blocks = required_blocks_per_seq * max(1, config.max_num_seqs)- if required_total_blocks > 0:- config.num_kvcache_blocks = min(config.num_kvcache_blocks, required_total_blocks)- if config.num_kvcache_blocks <= 0:+ config.num_kvcache_blocks = max(1, int(required_total_blocks))+ required_kv_bytes = config.num_kvcache_blocks * block_bytes+ free, total = torch.cuda.mem_get_info()+ current = torch.cuda.memory_stats()["allocated_bytes.all.current"]+ target_total_usage = total * config.gpu_memory_utilization+ allowed_kv_bytes = max(0, target_total_usage - current)+ if required_kv_bytes > allowed_kv_bytes:raise RuntimeError(- f"Insufficient GPU memory for KV cache. "+ f"Insufficient GPU memory for strict KV cache sizing under gpu_memory_utilization={config.gpu_memory_utilization:.2f}. "+ f"Required KV: {required_kv_bytes / 1024**3:.2f} GB "+ f"(blocks={config.num_kvcache_blocks}, block={block_bytes / 1024**2:.2f} MB), "+ f"Allowed by limit: {allowed_kv_bytes / 1024**3:.2f} GB, "f"Free: {free / 1024**3:.2f} GB, Current: {current / 1024**3:.2f} GB, "- f"Available for KV: {available_for_kv_cache / 1024**3:.2f} GB, "- f"Block size: {block_bytes / 1024**2:.2f} MB"+ f"Requested max_model_len={config.max_model_len}, max_num_seqs={config.max_num_seqs}.")- self.kv_cache = torch.empty(- 2,- hf_config.num_hidden_layers,- config.num_kvcache_blocks,- self.block_size,- num_kv_heads,- head_dim,- device="cuda",- dtype=self.dtype,- )+ try:+ self.kv_cache = torch.empty(+ 2,+ hf_config.num_hidden_layers,+ config.num_kvcache_blocks,+ self.block_size,+ num_kv_heads,+ head_dim,+ device="cuda",+ dtype=self.dtype,+ )+ except RuntimeError as exc:+ if "out of memory" in str(exc).lower():+ free_now, total_now = torch.cuda.mem_get_info()+ raise RuntimeError(+ f"Failed to allocate strict KV cache ({required_kv_bytes / 1024**3:.2f} GB) for "+ f"max_model_len={config.max_model_len}, max_num_seqs={config.max_num_seqs}. "+ f"Current free VRAM: {free_now / 1024**3:.2f} GB / {total_now / 1024**3:.2f} GB total."+ ) from exc+ raiselayer_id = 0for module in self.model.modules():if hasattr(module, "k_cache") and hasattr(module, "v_cache"):@@ -663,6 +786,7 @@ class ModelRunner:top_ks=top_ks if top_ks is not None else None,top_ps=top_ps if top_ps is not None else None,repetition_penalties=None, # Already applied above+ generator=self._sampling_generator,# input_ids=cond_input_ids,).tolist()@@ -735,6 +859,7 @@ class ModelRunner:top_ks=top_ks if top_ks is not None else None,top_ps=top_ps if top_ps is not None else None,repetition_penalties=None, # Already applied above+ generator=self._sampling_generator,# input_ids=seq_input_ids,).tolist()@@ -753,6 +878,7 @@ class ModelRunner:def capture_cudagraph(self):config = self.configcache_key = (config.max_model_len, config.max_num_seqs)+ model_device = self._get_model_device()cached = self._graph_cache.get(cache_key)if cached is not None:current_sig = self._get_graph_capture_signature()@@ -765,16 +891,16 @@ class ModelRunner:self._graph_cache_order.remove(cache_key)self._graph_cache_order.append(cache_key)return- self._drop_graph_cache_entry(cache_key)+ self._drop_graph_cache_entry(cache_key)hf_config = config.hf_configmax_bs = min(self.config.max_num_seqs, 512)max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size- input_ids = torch.zeros(max_bs, dtype=torch.int64)- positions = torch.zeros(max_bs, dtype=torch.int64)- slot_mapping = torch.zeros(max_bs, dtype=torch.int32)- context_lens = torch.zeros(max_bs, dtype=torch.int32)- block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32)- outputs = torch.zeros(max_bs, hf_config.hidden_size)+ input_ids = torch.zeros(max_bs, dtype=torch.int64, device=model_device)+ positions = torch.zeros(max_bs, dtype=torch.int64, device=model_device)+ slot_mapping = torch.zeros(max_bs, dtype=torch.int32, device=model_device)+ context_lens = torch.zeros(max_bs, dtype=torch.int32, device=model_device)+ block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32, device=model_device)+ outputs = torch.zeros(max_bs, hf_config.hidden_size, device=model_device)self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16))self.graphs = {}self.graph_pool = Nonediff --git a/shared/llm_engines/nanovllm/layers/linear.py b/shared/llm_engines/nanovllm/layers/linear.pyindex 3476cbb..ab19766 100644--- a/shared/llm_engines/nanovllm/layers/linear.py+++ b/shared/llm_engines/nanovllm/layers/linear.py@@ -117,9 +117,20 @@ class LinearBase(nn.Module):def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):device = self.weight.device- shard = self._shard_weight(loaded_weight, loaded_shard_id).contiguous()- q = shard.to(device=device, dtype=torch.int8, non_blocking=True)- self.qweight_data = q+ shard = self._shard_weight(loaded_weight, loaded_shard_id)+ if self.qweight_t.numel() == 0:+ self.qweight_t = torch.empty(+ (self.input_size, self.output_size),+ dtype=torch.int8,+ device=device,+ )+ if shard.dtype != torch.int8:+ shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)+ else:+ shard = shard.to(device=device, non_blocking=True)+ # Store directly in transposed layout to avoid an extra full-size transpose pass+ # after loading.+ self.qweight_t.t().copy_(shard)shard_key = loaded_shard_id if loaded_shard_id is not None else 0self._quant_data_loaded.add(shard_key)@@ -141,15 +152,20 @@ class LinearBase(nn.Module):self.output_scale = loaded_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)def finalize_quantized(self):- if self.qweight_data.numel() == 0 or self.qweight_scale.numel() == 0:+ if self.qweight_scale.numel() == 0:returnif len(self._quant_data_loaded) < self._quant_expected_shards:returnif len(self._quant_scale_loaded) < self._quant_expected_shards:return- # Keep int8 weights in KxN layout for int8 GEMM paths.- self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()- self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)+ if self.qweight_data.numel() == 0 and self.qweight_t.numel() == 0:+ return+ if self.qweight_t.numel() == 0:+ # Fallback path for loaders that still filled qweight_data (older checkpoints+ # or custom loaders). Keep memory usage low by releasing qweight_data immediately+ # after conversion.+ self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()+ self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)self.qweight_scale_fp32 = self.qweight_scale.to(dtype=torch.float32)if not torch.is_tensor(self.input_scale):raise RuntimeError(f"Invalid input_scale type: {type(self.input_scale)}")@@ -186,6 +202,7 @@ class LinearBase(nn.Module):raise RuntimeError("quanto.qbytes_mm op unavailable for int8 fallback path")if self.qweight_t.numel() == 0 and self.qweight_data.numel() != 0:self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()+ self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)qweight = self.qweight_t.transpose(0, 1).contiguous()scales = self.qweight_scale_fp32if scales.numel() == 0:@@ -206,6 +223,8 @@ class LinearBase(nn.Module):device = self.qweight_scale.deviceelse:device = torch.device("cuda" if torch.cuda.is_available() else "cpu")+ if getattr(device, "type", None) == "meta":+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")if self.weight.numel() != 0:self.weight = nn.Parameter(torch.empty(0, device=device, dtype=torch.bfloat16), requires_grad=False)@@ -301,12 +320,15 @@ class MergedColumnParallelLinear(ColumnParallelLinear):def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):device = self.weight.device- if self.qweight_data.numel() == 0:- self.qweight_data = torch.empty((self.output_size, self.input_size), dtype=torch.int8, device=device)+ if self.qweight_t.numel() == 0:+ self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)shard_offset, shard_size = self._merged_shard_meta(int(loaded_shard_id))- param_slice = self.qweight_data.narrow(0, shard_offset, shard_size)shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]- param_slice.copy_(shard.to(device=device, dtype=torch.int8, non_blocking=True))+ if shard.dtype != torch.int8:+ shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)+ else:+ shard = shard.to(device=device, non_blocking=True)+ self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())self._quant_data_loaded.add(int(loaded_shard_id))def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):@@ -363,12 +385,15 @@ class QKVParallelLinear(ColumnParallelLinear):def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):device = self.weight.device- if self.qweight_data.numel() == 0:- self.qweight_data = torch.empty((self.output_size, self.input_size), dtype=torch.int8, device=device)+ if self.qweight_t.numel() == 0:+ self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)shard_offset, shard_size = self._qkv_offset_size(str(loaded_shard_id))- param_slice = self.qweight_data.narrow(0, shard_offset, shard_size)shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]- param_slice.copy_(shard.to(device=device, dtype=torch.int8, non_blocking=True))+ if shard.dtype != torch.int8:+ shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)+ else:+ shard = shard.to(device=device, non_blocking=True)+ self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())self._quant_data_loaded.add(str(loaded_shard_id))def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):diff --git a/shared/llm_engines/nanovllm/layers/rotary_embedding.py b/shared/llm_engines/nanovllm/layers/rotary_embedding.pyindex 998d116..acaeffd 100644--- a/shared/llm_engines/nanovllm/layers/rotary_embedding.py+++ b/shared/llm_engines/nanovllm/layers/rotary_embedding.py@@ -25,14 +25,30 @@ class RotaryEmbedding(nn.Module):) -> None:super().__init__()self.head_size = head_size+ self.rotary_dim = rotary_dim+ self.max_position_embeddings = max_position_embeddings+ self.base = float(base)assert rotary_dim == head_size- inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))- t = torch.arange(max_position_embeddings, dtype=torch.float)+ cache = self._build_cache(device=None)+ self.register_buffer("cos_sin_cache", cache, persistent=False)++ def _build_cache(self, device: torch.device | str | None) -> torch.Tensor:+ inv_freq = 1.0 / (+ self.base ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32, device=device) / self.rotary_dim)+ )+ t = torch.arange(self.max_position_embeddings, dtype=torch.float32, device=device)freqs = torch.einsum("i,j -> ij", t, inv_freq)cos = freqs.cos()sin = freqs.sin()- cache = torch.cat((cos, sin), dim=-1).unsqueeze_(1)- self.register_buffer("cos_sin_cache", cache, persistent=False)+ return torch.cat((cos, sin), dim=-1).unsqueeze_(1)++ def materialize_cache(self, device: torch.device | str | None = None):+ target_device = device+ if target_device is None:+ target_device = self.cos_sin_cache.device+ if getattr(target_device, "type", None) == "meta":+ target_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")+ self.cos_sin_cache = self._build_cache(target_device)@torch.compiledef forward(diff --git a/shared/llm_engines/nanovllm/layers/sampler.py b/shared/llm_engines/nanovllm/layers/sampler.pyindex 41fcb27..8753882 100644--- a/shared/llm_engines/nanovllm/layers/sampler.py+++ b/shared/llm_engines/nanovllm/layers/sampler.py@@ -97,6 +97,7 @@ class Sampler(nn.Module):top_ps: Optional[torch.Tensor] = None,repetition_penalties: Optional[torch.Tensor] = None,input_ids: Optional[torch.Tensor] = None,+ generator: Optional[torch.Generator] = None,):"""Sample tokens from logits with optional top-k and top-p filtering.@@ -118,5 +119,6 @@ class Sampler(nn.Module):if invalid_rows.any():logits[invalid_rows, 0] = 0.0probs = torch.softmax(logits, dim=-1)- sample_tokens = probs.div_(torch.empty_like(probs).exponential_(1).clamp_min_(1e-10)).argmax(dim=-1)+ noise = torch.empty_like(probs).exponential_(1, generator=generator).clamp_min_(1e-10)+ sample_tokens = probs.div_(noise).argmax(dim=-1)return sample_tokensdiff --git a/shared/llm_engines/nanovllm/models/__init__.py b/shared/llm_engines/nanovllm/models/__init__.pynew file mode 100644index 0000000..692e251--- /dev/null+++ b/shared/llm_engines/nanovllm/models/__init__.py@@ -0,0 +1,52 @@+import importlib+from typing import Iterable+from torch import nn+++_ARCH_REGISTRY = {+ "Qwen3ForCausalLM": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),+}++_MODEL_TYPE_REGISTRY = {+ "qwen3": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),+}+++def _load_model_cls(module_name: str, class_name: str):+ module = importlib.import_module(module_name)+ model_cls = getattr(module, class_name, None)+ if model_cls is None:+ raise RuntimeError(f"Model class '{class_name}' was not found in '{module_name}'.")+ if not isinstance(model_cls, type) or not issubclass(model_cls, nn.Module):+ raise RuntimeError(f"Resolved '{module_name}.{class_name}' is not a torch.nn.Module subclass.")+ return model_cls+++def _iter_architectures(hf_config) -> Iterable[str]:+ architectures = getattr(hf_config, "architectures", None)+ if isinstance(architectures, (list, tuple)):+ for architecture in architectures:+ if isinstance(architecture, str) and len(architecture) > 0:+ yield architecture+++def resolve_model_class(hf_config):+ for architecture in _iter_architectures(hf_config):+ entry = _ARCH_REGISTRY.get(architecture, None)+ if entry is not None:+ return _load_model_cls(entry[0], entry[1])++ model_type = str(getattr(hf_config, "model_type", "")).strip().lower()+ if model_type:+ entry = _MODEL_TYPE_REGISTRY.get(model_type, None)+ if entry is not None:+ return _load_model_cls(entry[0], entry[1])++ supported_architectures = ", ".join(sorted(_ARCH_REGISTRY.keys()))+ supported_model_types = ", ".join(sorted(_MODEL_TYPE_REGISTRY.keys()))+ found_architectures = ", ".join(list(_iter_architectures(hf_config))) or "<none>"+ raise RuntimeError(+ "nanovllm does not support this model architecture. "+ f"Found architectures: {found_architectures}; model_type: '{model_type or '<none>'}'. "+ f"Supported architectures: {supported_architectures}; supported model_type values: {supported_model_types}."+ )diff --git a/shared/llm_engines/nanovllm/models/qwen3.py b/shared/llm_engines/nanovllm/models/qwen3.pyindex 3f1c845..7b98783 100644--- a/shared/llm_engines/nanovllm/models/qwen3.py+++ b/shared/llm_engines/nanovllm/models/qwen3.py@@ -204,8 +204,6 @@ class Qwen3ForCausalLM(nn.Module):super().__init__()self.model = Qwen3Model(config)self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)- if config.tie_word_embeddings:- self.lm_head.weight.data = self.model.embed_tokens.weight.data# Proxy attributes for weight loading compatibility# Some model weights use "embed_tokens" instead of "model.embed_tokens"diff --git a/shared/llm_engines/nanovllm/utils/loader.py b/shared/llm_engines/nanovllm/utils/loader.pyindex 17ac800..92bf8b1 100644--- a/shared/llm_engines/nanovllm/utils/loader.py+++ b/shared/llm_engines/nanovllm/utils/loader.py@@ -1,4 +1,6 @@+import gcimport os+from collections import OrderedDictfrom glob import globimport torchfrom torch import nn@@ -6,52 +8,99 @@ from safetensors import safe_opendef default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor):+ if param.is_meta:+ raise RuntimeError("default_weight_loader cannot copy into meta parameter directly.")+ if loaded_weight.device != param.device or loaded_weight.dtype != param.dtype:+ loaded_weight = loaded_weight.to(+ device=param.device,+ dtype=param.dtype,+ non_blocking=True,+ )param.data.copy_(loaded_weight)-def _get_parameter_safe(model: nn.Module, weight_name: str):- """- Try to get parameter from model, handling name mismatches.+def _coerce_loaded_weight_to_param(param: nn.Parameter, loaded_weight: torch.Tensor) -> torch.Tensor:+ target_device = param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")+ if loaded_weight.device != target_device:+ loaded_weight = loaded_weight.to(device=target_device, non_blocking=True)+ if loaded_weight.dtype != param.dtype:+ loaded_weight = loaded_weight.to(dtype=param.dtype, non_blocking=True)+ return loaded_weight- Some models have nested structure (e.g., Qwen3ForCausalLM has model.embed_tokens)- but weight files may have flat names (embed_tokens.weight).- """- # Try direct access first- try:- return model.get_parameter(weight_name)- except AttributeError:- pass- # Try with 'model.' prefix (for nested model structure)- try:- prefixed_name = f"model.{weight_name}"- return model.get_parameter(prefixed_name)- except AttributeError:- pass+def _restore_param_attributes(new_param: nn.Parameter, src_param: nn.Parameter) -> None:+ for attr_name, attr_value in src_param.__dict__.items():+ setattr(new_param, attr_name, attr_value)- # Try removing 'model.' prefix++def _replace_meta_parameter(+ parent: nn.Module,+ param_name: str,+ loaded_weight: torch.Tensor,+ src_param: nn.Parameter,+) -> nn.Parameter:+ new_param = nn.Parameter(+ loaded_weight,+ requires_grad=src_param.requires_grad,+ )+ parent._parameters[param_name] = new_param+ _restore_param_attributes(new_param, src_param)+ return new_param+++def _replace_tied_meta_parameter(+ model: nn.Module,+ old_param: nn.Parameter,+ loaded_weight: torch.Tensor,+) -> nn.Parameter:+ new_param = nn.Parameter(+ loaded_weight,+ requires_grad=old_param.requires_grad,+ )+ _restore_param_attributes(new_param, old_param)+ for module in model.modules():+ for name, param in list(module._parameters.items()):+ if param is old_param:+ module._parameters[name] = new_param+ return new_param+++def _should_replace_param_globally(model: nn.Module, param: nn.Parameter) -> bool:+ tied_ids = getattr(model, "_vllm_tied_param_ids", None)+ return bool(tied_ids) and id(param) in tied_ids+++def _candidate_weight_names(weight_name: str):+ candidates = [weight_name]if weight_name.startswith("model."):- try:- unprefixed_name = weight_name[6:] # Remove 'model.' prefix- return model.get_parameter(unprefixed_name)- except AttributeError:- pass+ candidates.append(weight_name[6:])+ else:+ candidates.append(f"model.{weight_name}")+ return candidates++def _get_parameter_target(model: nn.Module, weight_name: str):+ for candidate in _candidate_weight_names(weight_name):+ if "." in candidate:+ parent_name, param_name = candidate.rsplit(".", 1)+ else:+ parent_name, param_name = "", candidate+ try:+ parent = model if parent_name == "" else model.get_submodule(parent_name)+ except Exception:+ continue+ param = parent._parameters.get(param_name)+ if param is not None and isinstance(param, nn.Parameter):+ return parent, param_name, paramreturn Nonedef _get_submodule_safe(model: nn.Module, module_name: str):- try:- return model.get_submodule(module_name)- except Exception:- pass- try:- return model.get_submodule(f"model.{module_name}")- except Exception:- pass- if module_name.startswith("model."):+ for candidate in _candidate_weight_names(module_name):+ if candidate == "":+ return modeltry:- return model.get_submodule(module_name[6:])+ return model.get_submodule(candidate)except Exception:passreturn None@@ -70,60 +119,86 @@ class WeightStore:self.files = _list_safetensor_files(path)if not self.files:raise FileNotFoundError(f"No .safetensors files found in {path}")- self._file_handles = {}self._weight_to_file = {}+ self._file_to_weight_names = OrderedDict()self._pinned_weights = {}self.is_quanto_int8 = Falsefor file in self.files:- with safe_open(file, "pt", "cpu") as f:+ with safe_open(file, "pt", "cpu", writable_tensors=False) as f:for key in f.keys():self._weight_to_file[key] = file+ self._file_to_weight_names.setdefault(file, []).append(key)if key.endswith(".weight._data"):self.is_quanto_int8 = Trueif self.mode == "pinned":self._preload_pinned()- def _get_handle(self, file_path: str):- handle = self._file_handles.get(file_path)- if handle is None:- handle = safe_open(file_path, "pt", "cpu")- self._file_handles[file_path] = handle- return handle-def _preload_pinned(self):- for key, file in self._weight_to_file.items():- handle = self._get_handle(file)- tensor = handle.get_tensor(key)- if tensor.device.type != "cpu":- tensor = tensor.cpu()- if not tensor.is_pinned():- tensor = tensor.pin_memory()- self._pinned_weights[key] = tensor+ for file, keys in self._file_to_weight_names.items():+ with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:+ for key in keys:+ tensor = handle.get_tensor(key)+ if tensor.device.type != "cpu":+ tensor = tensor.cpu()+ if not tensor.is_pinned():+ tensor = tensor.pin_memory()+ self._pinned_weights[key] = tensor++ def close(self):+ self._weight_to_file.clear()+ self._file_to_weight_names.clear()+ self._pinned_weights.clear()++ def __del__(self):+ try:+ self.close()+ except Exception:+ passdef get_tensor(self, key: str) -> torch.Tensor:if self.mode == "pinned":return self._pinned_weights[key]- handle = self._get_handle(self._weight_to_file[key])- return handle.get_tensor(key)+ file = self._weight_to_file.get(key)+ if file is None:+ raise KeyError(key)+ with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:+ return handle.get_tensor(key)-def load_model(model: nn.Module, path: str, weight_store: WeightStore | None = None):+def load_model(+ model: nn.Module,+ path: str,+ weight_store: WeightStore | None = None,+ clone_loaded_tensors: bool = False,+):packed_modules_mapping = getattr(model, "packed_modules_mapping", {})if weight_store is None:safetensor_files = _list_safetensor_files(path)if not safetensor_files:raise FileNotFoundError(f"No .safetensors files found in {path}")for file in safetensor_files:- with safe_open(file, "pt", "cpu") as f:+ with safe_open(file, "pt", "cpu", writable_tensors=False) as f:for weight_name in f.keys():tensor = f.get_tensor(weight_name)_apply_weight(model, packed_modules_mapping, weight_name, tensor)_finalize_quantized_modules(model)return-- for weight_name in weight_store._weight_to_file.keys():- tensor = weight_store.get_tensor(weight_name)- _apply_weight(model, packed_modules_mapping, weight_name, tensor)+ for file in weight_store.files:+ file_weights = weight_store._file_to_weight_names.get(file, [])+ if not file_weights:+ continue+ with safe_open(file, "pt", "cpu", writable_tensors=False) as f:+ for weight_name in file_weights:+ tensor = f.get_tensor(weight_name)+ if clone_loaded_tensors:+ tensor = tensor.clone()+ try:+ _apply_weight(model, packed_modules_mapping, weight_name, tensor)+ finally:+ del tensor+ if torch.cuda.is_available():+ torch.cuda.synchronize()+ gc.collect()_finalize_quantized_modules(model)@@ -138,79 +213,78 @@ def _apply_weight(model: nn.Module, packed_modules_mapping, weight_name: str, teelif weight_name.endswith(".output_scale"):quant_suffix = "output_scale"- for k in packed_modules_mapping:- if k in weight_name:- v, shard_id = packed_modules_mapping[k]- mapped_name = weight_name.replace(k, v)- if quant_suffix is not None:- module_name = mapped_name- if quant_suffix == "qdata":- module_name = mapped_name[: -len(".weight._data")]- elif quant_suffix == "qscale":- module_name = mapped_name[: -len(".weight._scale")]- elif quant_suffix == "input_scale":- module_name = mapped_name[: -len(".input_scale")]- elif quant_suffix == "output_scale":- module_name = mapped_name[: -len(".output_scale")]- module = _get_submodule_safe(model, module_name)- if module is None:- print(f"[loader] Warning: Module not found: {module_name}")- return- if quant_suffix == "qdata":- loader_fn = getattr(module, "quant_weight_data_loader", None)- elif quant_suffix == "qscale":- loader_fn = getattr(module, "quant_weight_scale_loader", None)- elif quant_suffix == "input_scale":- loader_fn = getattr(module, "quant_input_scale_loader", None)- else:- loader_fn = getattr(module, "quant_output_scale_loader", None)- if loader_fn is None:- print(f"[loader] Warning: Quant loader not found on module: {module_name}")- return- if quant_suffix in ("qdata", "qscale"):- loader_fn(tensor, shard_id)- else:- loader_fn(tensor)- return- param_name = mapped_name- param = _get_parameter_safe(model, param_name)- if param is None:- print(f"[loader] Warning: Parameter not found: {param_name}")- return- weight_loader = getattr(param, "weight_loader")- weight_loader(param, tensor, shard_id)- return+ mapped_name = weight_name+ shard_id = None+ for src_name, mapped in packed_modules_mapping.items():+ if src_name in weight_name:+ mapped_target = mapped[0] if isinstance(mapped, tuple) else mapped+ mapped_name = weight_name.replace(src_name, mapped_target)+ shard_id = mapped[1] if isinstance(mapped, tuple) and len(mapped) > 1 else None+ break+if quant_suffix is not None:if quant_suffix == "qdata":- module_name = weight_name[: -len(".weight._data")]+ module_name = mapped_name[:-len(".weight._data")]+ loader_name = "quant_weight_data_loader"elif quant_suffix == "qscale":- module_name = weight_name[: -len(".weight._scale")]+ module_name = mapped_name[:-len(".weight._scale")]+ loader_name = "quant_weight_scale_loader"elif quant_suffix == "input_scale":- module_name = weight_name[: -len(".input_scale")]+ module_name = mapped_name[:-len(".input_scale")]+ loader_name = "quant_input_scale_loader"else:- module_name = weight_name[: -len(".output_scale")]+ module_name = mapped_name[:-len(".output_scale")]+ loader_name = "quant_output_scale_loader"module = _get_submodule_safe(model, module_name)if module is None:print(f"[loader] Warning: Module not found: {module_name}")return- if quant_suffix == "qdata":- loader_fn = getattr(module, "quant_weight_data_loader", None)- elif quant_suffix == "qscale":- loader_fn = getattr(module, "quant_weight_scale_loader", None)- elif quant_suffix == "input_scale":- loader_fn = getattr(module, "quant_input_scale_loader", None)- else:- loader_fn = getattr(module, "quant_output_scale_loader", None)+ loader_fn = getattr(module, loader_name, None)if loader_fn is None:print(f"[loader] Warning: Quant loader not found on module: {module_name}")return- loader_fn(tensor)+ if shard_id is not None and quant_suffix in ("qdata", "qscale"):+ loader_fn(tensor, shard_id)+ else:+ loader_fn(tensor)+ return++ target = _get_parameter_target(model, mapped_name)+ if target is None:+ print(f"[loader] Warning: Parameter not found: {mapped_name}")return- param = _get_parameter_safe(model, weight_name)- if param is None:- print(f"[loader] Warning: Parameter not found: {weight_name}")+ parent, param_name, param = target+ weight_loader = getattr(param, "weight_loader", None)++ if shard_id is not None and weight_loader is not None:+ if param.is_meta:+ replacement_device = (+ param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")+ )+ replacement = torch.empty(param.shape, device=replacement_device, dtype=param.dtype)+ if _should_replace_param_globally(model, param):+ param = _replace_tied_meta_parameter(model, param, replacement)+ else:+ param = _replace_meta_parameter(parent, param_name, replacement, param)+ try:+ weight_loader(param, tensor, shard_id)+ except TypeError:+ weight_loader(param, tensor)+ return++ if param.is_meta:+ loaded_weight = _coerce_loaded_weight_to_param(param, tensor)+ if loaded_weight.shape != param.shape:+ raise RuntimeError(+ f"Weight shape mismatch for '{mapped_name}': expected {tuple(param.shape)}, got {tuple(loaded_weight.shape)}"+ )+ if _should_replace_param_globally(model, param):+ _replace_tied_meta_parameter(model, param, loaded_weight)+ else:+ _replace_meta_parameter(parent, param_name, loaded_weight, param)return- weight_loader = getattr(param, "weight_loader", default_weight_loader)+ if weight_loader is None:+ weight_loader = default_weight_loaderweight_loader(param, tensor)diff --git a/shared/utils/transformers_fast_tokenizer_patch.py b/shared/utils/transformers_fast_tokenizer_patch.pyindex c1f0b8a..21c020d 100644--- a/shared/utils/transformers_fast_tokenizer_patch.py+++ b/shared/utils/transformers_fast_tokenizer_patch.py@@ -6,6 +6,7 @@ import sys_PATCH_ALLOWED_PATHS = None_ORIG_FAST_INIT = None+_DISABLE_FULL_TOKENIZER_PICKLE_CACHE = True # Temporary: disable Python pickle tokenizer cache load/dump.def _normalize_path(path):@@ -273,6 +274,8 @@ def _meta_matches(meta, tokenizer_dir):def _load_full_tokenizer_cache(tokenizer_dir, cache_tag=None):+ if _DISABLE_FULL_TOKENIZER_PICKLE_CACHE:+ return Nonecache_file, meta_file = _cache_paths(tokenizer_dir, cache_tag=cache_tag)if not os.path.isfile(cache_file) or not os.path.isfile(meta_file):return None@@ -287,6 +290,8 @@ def _load_full_tokenizer_cache(tokenizer_dir, cache_tag=None):def _save_full_tokenizer_cache(tokenizer_dir, tokenizer, cache_tag=None):+ if _DISABLE_FULL_TOKENIZER_PICKLE_CACHE:+ returncache_file, meta_file = _cache_paths(tokenizer_dir, cache_tag=cache_tag)meta = {"py_version": list(sys.version_info[:3]),diff --git a/wgp.py b/wgp.pyindex 376c89d..e44c2e6 100644--- a/wgp.py+++ b/wgp.py@@ -94,7 +94,7 @@ AUTOSAVE_TEMPLATE_PATH = AUTOSAVE_FILENAMECONFIG_FILENAME = "wgp_config.json"PROMPT_VARS_MAX = 10target_mmgp_version = "3.7.4"-WanGP_version = "10.83"+WanGP_version = "10.84"settings_version = 2.51max_source_video_frames = 3000prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None@@ -170,13 +170,6 @@ def clear_gen_cache():def release_model():global wan_model, offloadobj, reload_needed- if wan_model is not None:- close_fn = getattr(wan_model, "close", None)- if callable(close_fn):- try:- close_fn()- except Exception:- passwan_model = Noneclear_gen_cache()if "_cache" in offload.shared_state:@@ -3544,6 +3537,7 @@ def setup_prompt_enhancer(pipe, kwargs):fl.locate_file("Llama3_2/Llama3_2_quanto_bf16_int8.safetensors"),defaultConfigPath=fl.locate_file("Llama3_2/config.json", error_if_none=False),configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},+ writable_tensors=False,)prompt_enhancer_llm_model._validate_model_kwargs = lambda *_args, **_kwargs: Noneprompt_enhancer_llm_model._offload_hooks = ["generate"]@@ -3559,6 +3553,7 @@ def setup_prompt_enhancer(pipe, kwargs):forcedConfigPath=fl.locate_file("llama-joycaption-beta-one-hf-llava/llama_config.json", error_if_none=False),configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},preprocess_sd=preprocess_sd,+ writable_tensors=False,)prompt_enhancer_llm_tokenizer = AutoTokenizer.from_pretrained(fl.locate_folder("llama-joycaption-beta-one-hf-llava"))@@ -5779,7 +5774,7 @@ def generate_video(torch.set_grad_enabled(False)if mode.startswith("edit_"):edit_video(send_cmd, state, mode, video_source, seed, temporal_upsampling, spatial_upsampling, film_grain_intensity, film_grain_saturation, MMAudio_setting, MMAudio_prompt, MMAudio_neg_prompt, repeat_generation, audio_source)- return+ return Truewith lock:file_list = gen["file_list"]file_settings_list = gen["file_settings_list"]@@ -5833,7 +5828,6 @@ def generate_video(profile = compute_profile(override_profile, output_type)enhancer_mode = server_config.get("enhancer_mode", 1)if model_type != transformer_type or reload_needed or profile != loaded_profile:- wan_model = Nonerelease_model()send_cmd("status", f"Loading model {get_model_name(model_type)}...")wan_model, offloadobj = load_models(@@ -5846,7 +5840,7 @@ def generate_video(reload_needed= Falseif args.test:send_cmd("info", "Test mode: model loaded, skipping generation.")- return+ return Trueoverridden_attention = override_attention if len(override_attention) else get_overridden_attention(model_type)# if overridden_attention is not None and overridden_attention != attention_mode: print(f"Attention mode has been overriden to {overridden_attention} for model type '{model_type}'")attn = overridden_attention if overridden_attention is not None else attention_mode@@ -5855,7 +5849,7 @@ def generate_video(elif not attn in attention_modes_supported:send_cmd("info", f"You have selected attention mode '{attention_mode}'. However it is not installed or supported on your system. You should either install it or switch to the default 'sdpa' attention.")send_cmd("exit")- return+ return Truewidth, height = resolution.split("x")width, height = int(width) // block_size * block_size, int(height) // block_size * block_size@@ -6135,7 +6129,7 @@ def generate_video(wan_model._interrupt = Falseabort = Falseif gen.get("abort", False):- return+ return True# gen["abort"] = Falsegen["prompt"] = promptrepeat_no = 0@@ -6663,7 +6657,7 @@ def generate_video(print('\n'.join(tb))send_cmd("error", new_error)clear_status(state)- return+ return Falsesrc_video = src_video2 = src_mask = src_mask2 = Noneif skip_steps_cache != None :skip_steps_cache.previous_residual = None@@ -6950,6 +6944,7 @@ def generate_video(cleanup_temp_audio_files(control_audio_tracks + source_audio_tracks)remove_temp_filenames(temp_filenames_list)+ return Truedef prepare_generate_video(state):@@ -7129,7 +7124,7 @@ def process_tasks(state):filtered_params = {k: v for k, v in params.items() if k in expected_args}plugin_data = task.pop('plugin_data', {})- generate_video(task, send_cmd, plugin_data=plugin_data, **filtered_params)+ success = generate_video(task, send_cmd, plugin_data=plugin_data, **filtered_params)except Exception as e:tb = traceback.format_exc().split('\n')[:-1]@@ -7145,7 +7140,7 @@ def process_tasks(state):gen["early_stop"] = Falsegen["early_stop_forwarded"] = False-+ if not success: breakwith lock:queue[:] = [item for item in queue if item['id'] != task_id]update_global_queue_ref(queue)@@ -8819,7 +8814,6 @@ def preload_model_when_switching(state):if "S" in preload_model_policy:model_type = get_state_model_type(state)if model_type != transformer_type:- wan_model = Nonerelease_model()model_filename = get_model_name(model_type)yield f"Loading model {model_filename}..."@@ -8836,7 +8830,6 @@ def unload_model_if_needed(state):global wan_modelif "U" in preload_model_policy:if wan_model != None:- wan_model = Nonerelease_model()def all_letters(source_str, letters):