Skip to content
Python

RAM optimisations

DeepBeepMeepf5bf0259359d
diff --git a/README.md b/README.md
index ad8ed56..e1d9ced 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@ WanGP supports the Wan (and derived models) but also Hunyuan Video, Flux, Qwen,
## 🔥 Latest Updates :
-### February 10th 2026: WanGP v10.83, Easy Metal
+### February 12th 2026: WanGP v10.84, Easy Metal
- **Ace Step 1.5 Turbo Super Charged**: all the best features of *Ace Step 1.5* are now in *WanGP* and are *Fast* & *Easy* to use:
* Manual Selection of *Bpm*, *Keyscale*, *Time Signature* & *Language*
@@ -58,7 +58,8 @@ Improved queue reordering: items can now be dragged and dropped directly onto th
*update 10.81*: Fixes\
*update 10.82*: UI update\
-*update 10.83*: Kugel Audio Split
+*update 10.83*: Kugel Audio Split\
+*update 10.84*: Ace Step RAM optimizations (fixed memory leak & reduce RAM requirements)
**Note to RTX 50xx owners**: you will need to upgrade to *pytorch 2.10* (see upgrade procedure below) to be able to use *Triton*
diff --git a/models/TTS/ace_step/music_dcae/music_dcae_pipeline.py b/models/TTS/ace_step/music_dcae/music_dcae_pipeline.py
index e28f92b..8115687 100644
--- a/models/TTS/ace_step/music_dcae/music_dcae_pipeline.py
+++ b/models/TTS/ace_step/music_dcae/music_dcae_pipeline.py
@@ -42,6 +42,7 @@ class MusicDCAE(ModelMixin, ConfigMixin, FromOriginalModelMixin):
dcae_weights_path,
modelClass=AutoencoderDC,
defaultConfigPath=dcae_config_path,
+ writable_tensors=False,
default_dtype=self._dtype,
ignore_unused_weights=True,
)
@@ -51,6 +52,7 @@ class MusicDCAE(ModelMixin, ConfigMixin, FromOriginalModelMixin):
vocoder_weights_path,
modelClass=ADaMoSHiFiGANV1,
defaultConfigPath=vocoder_config_path,
+ writable_tensors=False,
default_dtype=self._dtype,
ignore_unused_weights=True,
)
diff --git a/models/TTS/ace_step/pipeline_ace_step.py b/models/TTS/ace_step/pipeline_ace_step.py
index 09d44fd..a1c6009 100644
--- a/models/TTS/ace_step/pipeline_ace_step.py
+++ b/models/TTS/ace_step/pipeline_ace_step.py
@@ -158,6 +158,7 @@ class ACEStepPipeline:
self.transformer_weights_path,
modelClass=ACEStepTransformer2DModel,
defaultConfigPath=self.transformer_config_path,
+ writable_tensors=False,
default_dtype=self.dtype,
ignore_unused_weights=True,
)
@@ -193,6 +194,7 @@ class ACEStepPipeline:
self.text_encoder_weights_path,
modelClass=UMT5EncoderModel,
defaultConfigPath=self.text_encoder_config_path,
+ writable_tensors=False,
default_dtype=self.dtype,
ignore_unused_weights=True,
preprocess_sd=_fix_umt5_state_dict,
diff --git a/models/TTS/ace_step15/models/autoencoder_oobleck.py b/models/TTS/ace_step15/models/autoencoder_oobleck.py
new file mode 100644
index 0000000..299ce97
--- /dev/null
+++ b/models/TTS/ace_step15/models/autoencoder_oobleck.py
@@ -0,0 +1,455 @@
+# Copyright 2025 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+from dataclasses import dataclass
+from typing import Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.nn.utils.parametrizations import weight_norm
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.models.autoencoders.vae import AutoencoderMixin
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.utils import BaseOutput
+from diffusers.utils.accelerate_utils import apply_forward_hook
+from diffusers.utils.torch_utils import randn_tensor
+
+
+class Snake1d(nn.Module):
+ """
+ A 1-dimensional Snake activation function module.
+ """
+
+ def __init__(self, hidden_dim, logscale=True):
+ super().__init__()
+ self.alpha = nn.Parameter(torch.zeros(1, hidden_dim, 1))
+ self.beta = nn.Parameter(torch.zeros(1, hidden_dim, 1))
+
+ self.alpha.requires_grad = True
+ self.beta.requires_grad = True
+ self.logscale = logscale
+
+ def forward(self, hidden_states, preserve_input: bool = False):
+ shape = hidden_states.shape
+ hidden_states = hidden_states.reshape(shape[0], shape[1], -1)
+
+ alpha = self.alpha if not self.logscale else torch.exp(self.alpha)
+ beta = self.beta if not self.logscale else torch.exp(self.beta)
+ inv_beta = (beta + 1e-9).reciprocal()
+
+ use_inplace = (not preserve_input) and (not torch.is_grad_enabled())
+ wave = torch.empty_like(hidden_states)
+ torch.mul(hidden_states, alpha, out=wave)
+ torch.sin_(wave)
+ wave.mul_(wave)
+ wave.mul_(inv_beta)
+ if use_inplace:
+ hidden_states.add_(wave)
+ return hidden_states.reshape(shape)
+ wave.add_(hidden_states)
+ return wave.reshape(shape)
+
+
+class OobleckResidualUnit(nn.Module):
+ """
+ A residual unit composed of Snake1d and weight-normalized Conv1d layers with dilations.
+ """
+
+ def __init__(self, dimension: int = 16, dilation: int = 1):
+ super().__init__()
+ pad = ((7 - 1) * dilation) // 2
+
+ self.snake1 = Snake1d(dimension)
+ self.conv1 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=7, dilation=dilation, padding=pad))
+ self.snake2 = Snake1d(dimension)
+ self.conv2 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=1))
+
+ def forward(self, hidden_state):
+ """
+ Forward pass through the residual unit.
+
+ Args:
+ hidden_state (`torch.Tensor` of shape `(batch_size, channels, time_steps)`):
+ Input tensor .
+
+ Returns:
+ output_tensor (`torch.Tensor` of shape `(batch_size, channels, time_steps)`)
+ Input tensor after passing through the residual unit.
+ """
+ output_tensor = hidden_state
+ output_tensor = self.conv1(self.snake1(output_tensor, preserve_input=True))
+ output_tensor = self.conv2(self.snake2(output_tensor))
+
+ padding = (hidden_state.shape[-1] - output_tensor.shape[-1]) // 2
+ if padding > 0:
+ hidden_state = hidden_state[..., padding:-padding]
+ output_tensor += hidden_state
+ return output_tensor
+
+
+class OobleckEncoderBlock(nn.Module):
+ """Encoder block used in Oobleck encoder."""
+
+ def __init__(self, input_dim, output_dim, stride: int = 1):
+ super().__init__()
+
+ self.res_unit1 = OobleckResidualUnit(input_dim, dilation=1)
+ self.res_unit2 = OobleckResidualUnit(input_dim, dilation=3)
+ self.res_unit3 = OobleckResidualUnit(input_dim, dilation=9)
+ self.snake1 = Snake1d(input_dim)
+ self.conv1 = weight_norm(
+ nn.Conv1d(input_dim, output_dim, kernel_size=2 * stride, stride=stride, padding=math.ceil(stride / 2))
+ )
+
+ def forward(self, hidden_state):
+ hidden_state = self.res_unit1(hidden_state)
+ hidden_state = self.res_unit2(hidden_state)
+ hidden_state = self.snake1(self.res_unit3(hidden_state))
+ hidden_state = self.conv1(hidden_state)
+
+ return hidden_state
+
+
+class OobleckDecoderBlock(nn.Module):
+ """Decoder block used in Oobleck decoder."""
+
+ def __init__(self, input_dim, output_dim, stride: int = 1):
+ super().__init__()
+
+ self.snake1 = Snake1d(input_dim)
+ self.conv_t1 = weight_norm(
+ nn.ConvTranspose1d(
+ input_dim,
+ output_dim,
+ kernel_size=2 * stride,
+ stride=stride,
+ padding=math.ceil(stride / 2),
+ )
+ )
+ self.res_unit1 = OobleckResidualUnit(output_dim, dilation=1)
+ self.res_unit2 = OobleckResidualUnit(output_dim, dilation=3)
+ self.res_unit3 = OobleckResidualUnit(output_dim, dilation=9)
+
+ def forward(self, hidden_state):
+ hidden_state = self.conv_t1(self.snake1(hidden_state, preserve_input=False))
+ hidden_state = self.res_unit1(hidden_state)
+ hidden_state = self.res_unit2(hidden_state)
+ return self.res_unit3(hidden_state)
+
+
+class OobleckDiagonalGaussianDistribution(object):
+ def __init__(self, parameters: torch.Tensor, deterministic: bool = False):
+ self.parameters = parameters
+ self.mean, self.scale = parameters.chunk(2, dim=1)
+ self.std = nn.functional.softplus(self.scale) + 1e-4
+ self.var = self.std * self.std
+ self.logvar = torch.log(self.var)
+ self.deterministic = deterministic
+
+ def sample(self, generator: Optional[torch.Generator] = None) -> torch.Tensor:
+ # make sure sample is on the same device as the parameters and has same dtype
+ sample = randn_tensor(
+ self.mean.shape,
+ generator=generator,
+ device=self.parameters.device,
+ dtype=self.parameters.dtype,
+ )
+ x = self.mean + self.std * sample
+ return x
+
+ def kl(self, other: "OobleckDiagonalGaussianDistribution" = None) -> torch.Tensor:
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ else:
+ if other is None:
+ return (self.mean * self.mean + self.var - self.logvar - 1.0).sum(1).mean()
+ else:
+ normalized_diff = torch.pow(self.mean - other.mean, 2) / other.var
+ var_ratio = self.var / other.var
+ logvar_diff = self.logvar - other.logvar
+
+ kl = normalized_diff + var_ratio + logvar_diff - 1
+
+ kl = kl.sum(1).mean()
+ return kl
+
+ def mode(self) -> torch.Tensor:
+ return self.mean
+
+
+@dataclass
+class AutoencoderOobleckOutput(BaseOutput):
+ """
+ Output of AutoencoderOobleck encoding method.
+
+ Args:
+ latent_dist (`OobleckDiagonalGaussianDistribution`):
+ Encoded outputs of `Encoder` represented as the mean and standard deviation of
+ `OobleckDiagonalGaussianDistribution`. `OobleckDiagonalGaussianDistribution` allows for sampling latents
+ from the distribution.
+ """
+
+ latent_dist: "OobleckDiagonalGaussianDistribution" # noqa: F821
+
+
+@dataclass
+class OobleckDecoderOutput(BaseOutput):
+ r"""
+ Output of decoding method.
+
+ Args:
+ sample (`torch.Tensor` of shape `(batch_size, audio_channels, sequence_length)`):
+ The decoded output sample from the last layer of the model.
+ """
+
+ sample: torch.Tensor
+
+
+class OobleckEncoder(nn.Module):
+ """Oobleck Encoder"""
+
+ def __init__(self, encoder_hidden_size, audio_channels, downsampling_ratios, channel_multiples):
+ super().__init__()
+
+ strides = downsampling_ratios
+ channel_multiples = [1] + channel_multiples
+
+ # Create first convolution
+ self.conv1 = weight_norm(nn.Conv1d(audio_channels, encoder_hidden_size, kernel_size=7, padding=3))
+
+ self.block = []
+ # Create EncoderBlocks that double channels as they downsample by `stride`
+ for stride_index, stride in enumerate(strides):
+ self.block += [
+ OobleckEncoderBlock(
+ input_dim=encoder_hidden_size * channel_multiples[stride_index],
+ output_dim=encoder_hidden_size * channel_multiples[stride_index + 1],
+ stride=stride,
+ )
+ ]
+
+ self.block = nn.ModuleList(self.block)
+ d_model = encoder_hidden_size * channel_multiples[-1]
+ self.snake1 = Snake1d(d_model)
+ self.conv2 = weight_norm(nn.Conv1d(d_model, encoder_hidden_size, kernel_size=3, padding=1))
+
+ def forward(self, hidden_state):
+ hidden_state = self.conv1(hidden_state)
+
+ for module in self.block:
+ hidden_state = module(hidden_state)
+
+ hidden_state = self.snake1(hidden_state)
+ hidden_state = self.conv2(hidden_state)
+
+ return hidden_state
+
+
+class OobleckDecoder(nn.Module):
+ """Oobleck Decoder"""
+
+ def __init__(self, channels, input_channels, audio_channels, upsampling_ratios, channel_multiples):
+ super().__init__()
+
+ strides = upsampling_ratios
+ channel_multiples = [1] + channel_multiples
+
+ # Add first conv layer
+ self.conv1 = weight_norm(nn.Conv1d(input_channels, channels * channel_multiples[-1], kernel_size=7, padding=3))
+
+ # Add upsampling + MRF blocks
+ block = []
+ for stride_index, stride in enumerate(strides):
+ block += [
+ OobleckDecoderBlock(
+ input_dim=channels * channel_multiples[len(strides) - stride_index],
+ output_dim=channels * channel_multiples[len(strides) - stride_index - 1],
+ stride=stride,
+ )
+ ]
+
+ self.block = nn.ModuleList(block)
+ output_dim = channels
+ self.snake1 = Snake1d(output_dim)
+ self.conv2 = weight_norm(nn.Conv1d(channels, audio_channels, kernel_size=7, padding=3, bias=False))
+
+ def forward(self, hidden_state):
+ hidden_state = self.conv1(hidden_state)
+
+ for layer in self.block:
+ hidden_state = layer(hidden_state)
+
+ return self.conv2(self.snake1(hidden_state, preserve_input=False))
+
+
+class AutoencoderOobleck(ModelMixin, AutoencoderMixin, ConfigMixin):
+ r"""
+ An autoencoder for encoding waveforms into latents and decoding latent representations into waveforms. First
+ introduced in Stable Audio.
+
+ This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented
+ for all models (such as downloading or saving).
+
+ Parameters:
+ encoder_hidden_size (`int`, *optional*, defaults to 128):
+ Intermediate representation dimension for the encoder.
+ downsampling_ratios (`List[int]`, *optional*, defaults to `[2, 4, 4, 8, 8]`):
+ Ratios for downsampling in the encoder. These are used in reverse order for upsampling in the decoder.
+ channel_multiples (`List[int]`, *optional*, defaults to `[1, 2, 4, 8, 16]`):
+ Multiples used to determine the hidden sizes of the hidden layers.
+ decoder_channels (`int`, *optional*, defaults to 128):
+ Intermediate representation dimension for the decoder.
+ decoder_input_channels (`int`, *optional*, defaults to 64):
+ Input dimension for the decoder. Corresponds to the latent dimension.
+ audio_channels (`int`, *optional*, defaults to 2):
+ Number of channels in the audio data. Either 1 for mono or 2 for stereo.
+ sampling_rate (`int`, *optional*, defaults to 44100):
+ The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz).
+ """
+
+ _supports_gradient_checkpointing = False
+ _supports_group_offloading = False
+
+ @register_to_config
+ def __init__(
+ self,
+ encoder_hidden_size=128,
+ downsampling_ratios=[2, 4, 4, 8, 8],
+ channel_multiples=[1, 2, 4, 8, 16],
+ decoder_channels=128,
+ decoder_input_channels=64,
+ audio_channels=2,
+ sampling_rate=44100,
+ ):
+ super().__init__()
+
+ self.encoder_hidden_size = encoder_hidden_size
+ self.downsampling_ratios = downsampling_ratios
+ self.decoder_channels = decoder_channels
+ self.upsampling_ratios = downsampling_ratios[::-1]
+ self.hop_length = int(np.prod(downsampling_ratios))
+ self.sampling_rate = sampling_rate
+
+ self.encoder = OobleckEncoder(
+ encoder_hidden_size=encoder_hidden_size,
+ audio_channels=audio_channels,
+ downsampling_ratios=downsampling_ratios,
+ channel_multiples=channel_multiples,
+ )
+
+ self.decoder = OobleckDecoder(
+ channels=decoder_channels,
+ input_channels=decoder_input_channels,
+ audio_channels=audio_channels,
+ upsampling_ratios=self.upsampling_ratios,
+ channel_multiples=channel_multiples,
+ )
+
+ self.use_slicing = False
+
+ @apply_forward_hook
+ def encode(
+ self, x: torch.Tensor, return_dict: bool = True
+ ) -> Union[AutoencoderOobleckOutput, Tuple[OobleckDiagonalGaussianDistribution]]:
+ """
+ Encode a batch of images into latents.
+
+ Args:
+ x (`torch.Tensor`): Input batch of images.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple.
+
+ Returns:
+ The latent representations of the encoded images. If `return_dict` is True, a
+ [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned.
+ """
+ if self.use_slicing and x.shape[0] > 1:
+ encoded_slices = [self.encoder(x_slice) for x_slice in x.split(1)]
+ h = torch.cat(encoded_slices)
+ else:
+ h = self.encoder(x)
+
+ posterior = OobleckDiagonalGaussianDistribution(h)
+
+ if not return_dict:
+ return (posterior,)
+
+ return AutoencoderOobleckOutput(latent_dist=posterior)
+
+ def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[OobleckDecoderOutput, torch.Tensor]:
+ dec = self.decoder(z)
+
+ if not return_dict:
+ return (dec,)
+
+ return OobleckDecoderOutput(sample=dec)
+
+ @apply_forward_hook
+ def decode(
+ self, z: torch.FloatTensor, return_dict: bool = True, generator=None
+ ) -> Union[OobleckDecoderOutput, torch.FloatTensor]:
+ """
+ Decode a batch of images.
+
+ Args:
+ z (`torch.Tensor`): Input batch of latent vectors.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether to return a [`~models.vae.OobleckDecoderOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.vae.OobleckDecoderOutput`] or `tuple`:
+ If return_dict is True, a [`~models.vae.OobleckDecoderOutput`] is returned, otherwise a plain `tuple`
+ is returned.
+
+ """
+ if self.use_slicing and z.shape[0] > 1:
+ decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)]
+ decoded = torch.cat(decoded_slices)
+ else:
+ decoded = self._decode(z).sample
+
+ if not return_dict:
+ return (decoded,)
+
+ return OobleckDecoderOutput(sample=decoded)
+
+ def forward(
+ self,
+ sample: torch.Tensor,
+ sample_posterior: bool = False,
+ return_dict: bool = True,
+ generator: Optional[torch.Generator] = None,
+ ) -> Union[OobleckDecoderOutput, torch.Tensor]:
+ r"""
+ Args:
+ sample (`torch.Tensor`): Input sample.
+ sample_posterior (`bool`, *optional*, defaults to `False`):
+ Whether to sample from the posterior.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`OobleckDecoderOutput`] instead of a plain tuple.
+ """
+ x = sample
+ posterior = self.encode(x).latent_dist
+ if sample_posterior:
+ z = posterior.sample(generator=generator)
+ else:
+ z = posterior.mode()
+ dec = self.decode(z).sample
+
+ if not return_dict:
+ return (dec,)
+
+ return OobleckDecoderOutput(sample=dec)
diff --git a/models/TTS/ace_step15/pipeline_ace_step15.py b/models/TTS/ace_step15/pipeline_ace_step15.py
index 3a06320..4bc9e2b 100644
--- a/models/TTS/ace_step15/pipeline_ace_step15.py
+++ b/models/TTS/ace_step15/pipeline_ace_step15.py
@@ -15,11 +15,11 @@ import torchaudio
import yaml
from tqdm import tqdm
from transformers import AutoTokenizer, Qwen3ForCausalLM, Qwen3Model
-from diffusers import AutoencoderOobleck
from mmgp import offload
from shared.utils.text_encoder_cache import TextEncoderCache
+from .models.autoencoder_oobleck import AutoencoderOobleck
from .models.ace_step15_hf import AceStepConditionGenerationModel
_DEFAULT_TIMBRE = [
@@ -51,6 +51,7 @@ _ACE_STEP15_DEFAULT_TIMESIGNATURE = 2
_ACE_STEP15_DEFAULT_KEYSCALE = "C major"
_ACE_STEP15_DURATION_MIN_SECONDS = 5
_ACE_STEP15_DURATION_MAX_SECONDS = 600
+_ACE_STEP15_PHASE1_MAX_TOKENS = 512
_ACE_STEP15_ALLOWED_TIMESIGNATURES = {2, 3, 4, 6}
_ACE_STEP15_BPM_OPTIONS = [str(v) for v in range(30, 301)]
_ACE_STEP15_TIMESIGNATURE_OPTIONS = ["2", "3", "4", "6"]
@@ -111,6 +112,7 @@ class ACEStep15Pipeline:
ignore_lm_cache_seed: bool = False,
lm_decoder_engine: str = "legacy",
lm_vllm_weight_mode: str = "lazy",
+ lm_vllm_free_mmap_after_cuda_copy: bool = True,
device=None,
dtype=torch.bfloat16,
):
@@ -139,6 +141,11 @@ class ACEStep15Pipeline:
if self.lm_engine not in ("legacy", "pt", "vllm"):
self.lm_engine = "legacy"
self.lm_vllm_weight_mode = (lm_vllm_weight_mode or "lazy").strip().lower()
+ if self.lm_engine == "vllm" and self.lm_vllm_weight_mode not in ("lazy", "pinned"):
+ raise ValueError(
+ f"Unsupported Ace Step vllm LM weight mode '{self.lm_vllm_weight_mode}'. Supported: lazy, pinned."
+ )
+ self.lm_vllm_free_mmap_after_cuda_copy = bool(lm_vllm_free_mmap_after_cuda_copy)
self._interrupt = False
self._early_stop = False
@@ -174,32 +181,45 @@ class ACEStep15Pipeline:
modelClass=AceStepConditionGenerationModel,
defaultConfigPath=transformer_config_path,
default_dtype=self.dtype,
+ writable_tensors=False,
ignore_unused_weights=True,
)
self.ace_step_transformer.eval()
self.model = self.ace_step_transformer
- self._patch_oobleck_weight_norm()
self.audio_vae = offload.fast_load_transformers_model(
vae_weights_path,
modelClass=AutoencoderOobleck,
defaultConfigPath=vae_config_path,
default_dtype=self.dtype,
+ writable_tensors=False,
ignore_unused_weights=True,
)
+ self._fold_oobleck_decoder_weight_norm_once(self.audio_vae)
self.audio_vae.eval()
self.audio_vae._offload_hooks = ["encode", "decode"]
self.audio_vae.get_VAE_tile_size = _ace_step15_get_vae_tile_size
self.vae = self.audio_vae
@staticmethod
- def _patch_oobleck_weight_norm():
+ def _fold_oobleck_decoder_weight_norm_once(audio_vae):
try:
- from torch.nn.utils import parametrizations
- from diffusers.models.autoencoders import autoencoder_oobleck
- autoencoder_oobleck.weight_norm = parametrizations.weight_norm
+ from torch.nn.utils import parametrize
except Exception:
return
+ decoder = getattr(audio_vae, "decoder", None)
+ if decoder is None:
+ return
+ folded = 0
+ with torch.no_grad():
+ for module in decoder.modules():
+ module_parametrizations = getattr(module, "parametrizations", None)
+ if module_parametrizations is None or "weight" not in module_parametrizations:
+ continue
+ parametrize.remove_parametrizations(module, "weight", leave_parametrized=True)
+ folded += 1
+ # if folded > 0:
+ # print(f"[ace_step15] Folded Oobleck decoder weight_norm once for {folded} layers.")
def _init_lm_hint_modules(self):
self._lm_hint_quantizer = None
@@ -239,6 +259,7 @@ class ACEStep15Pipeline:
self.lm_tokenizer = None
if not self.enable_lm:
return
+ print("Loading Tokenizers...")
lm_loader = lambda: AutoTokenizer.from_pretrained(
self.lm_tokenizer_dir,
local_files_only=True,
@@ -284,6 +305,7 @@ class ACEStep15Pipeline:
modelClass=Qwen3Model,
defaultConfigPath=config_path,
default_dtype=self.dtype,
+ writable_tensors=False,
ignore_unused_weights=True,
)
self.text_encoder_2.eval()
@@ -307,6 +329,7 @@ class ACEStep15Pipeline:
defaultConfigPath=config_path,
default_dtype=self.dtype,
preprocess_sd=_remap_lm_state_dict,
+ writable_tensors=False,
ignore_unused_weights=True,
)
self.lm_model.eval()
@@ -341,6 +364,7 @@ class ACEStep15Pipeline:
lm_weights_path=self.lm_weights_path,
audio_code_mask=getattr(self, "_audio_code_mask", None),
audio_code_token_map=getattr(self, "_audio_code_token_map", {}),
+ free_mmap_after_cuda_copy=self.lm_vllm_free_mmap_after_cuda_copy,
weight_load_mode=self.lm_vllm_weight_mode,
)
if self.lm_engine in ("pt", "vllm") and self._lm_engine_impl is None:
@@ -614,6 +638,7 @@ class ACEStep15Pipeline:
logits_processor_update_state=None,
stop_checker=None,
progress_label: str = "LM text",
+ release_vram_after: bool = True,
):
if self._lm_engine_impl is not None and hasattr(self._lm_engine_impl, "generate_text"):
return self._lm_engine_impl.generate_text(
@@ -631,6 +656,7 @@ class ACEStep15Pipeline:
logits_processor_update_state=logits_processor_update_state,
stop_checker=stop_checker,
progress_label=progress_label,
+ release_vram_after=release_vram_after,
)
if self.lm_engine != "legacy":
raise RuntimeError(f"LM engine '{self.lm_engine}' is not initialized for text generation.")
@@ -779,6 +805,7 @@ class ACEStep15Pipeline:
top_p,
top_k,
callback=None,
+ release_vram_after: bool = True,
):
phase1_metadata = dict(metadata)
if self._should_abort():
@@ -811,7 +838,7 @@ class ACEStep15Pipeline:
text_out = self._generate_lm_text(
prompt=prompt,
prompt_negative=prompt_negative,
- max_tokens=512,
+ max_tokens=_ACE_STEP15_PHASE1_MAX_TOKENS,
temperature=temperature,
top_p=top_p,
top_k=top_k,
@@ -822,6 +849,7 @@ class ACEStep15Pipeline:
logits_processor_update_state=processor.update_state,
stop_checker=None,
progress_label="LM Compute Metadata",
+ release_vram_after=release_vram_after,
)
if text_out is None:
raise RuntimeError("LM phase1 failed while inferring metadata.")
@@ -921,6 +949,43 @@ class ACEStep15Pipeline:
int(seed_value) if seed_value is not None else None,
)
+ def _reserve_vllm_runtime_for_max_duration(self, tags: str, lyrics: str, negative_prompt: str, lm_cfg_scale):
+ if self.lm_engine != "vllm" or self._lm_engine_impl is None or self.lm_tokenizer is None:
+ return
+ reserve_fn = getattr(self._lm_engine_impl, "reserve_runtime", None)
+ if not callable(reserve_fn):
+ return
+ cfg_phase1 = 1.0 if lm_cfg_scale is None else float(lm_cfg_scale)
+ cfg_phase2 = 2.5 if lm_cfg_scale is None else float(lm_cfg_scale)
+ reserve_cfg = max(1.0, cfg_phase1, cfg_phase2)
+ max_duration_seconds = int(_ACE_STEP15_DURATION_MAX_SECONDS)
+ phase1_max_tokens = int(_ACE_STEP15_PHASE1_MAX_TOKENS)
+ phase2_max_tokens = max(1, max_duration_seconds * 5)
+ reserve_max_tokens = max(phase1_max_tokens, phase2_max_tokens)
+
+ phase1_prompt = self._build_reference_phase1_prompt(tags, lyrics, is_negative_prompt=False, negative_prompt=negative_prompt)
+ phase1_prompt_negative = self._build_reference_phase1_prompt(tags, lyrics, is_negative_prompt=True, negative_prompt=negative_prompt)
+ reserve_metadata = {
+ "bpm": _ACE_STEP15_DEFAULT_BPM,
+ "duration": max_duration_seconds,
+ "keyscale": _ACE_STEP15_DEFAULT_KEYSCALE,
+ "timesignature": _ACE_STEP15_DEFAULT_TIMESIGNATURE,
+ "caption": tags,
+ }
+ reserve_cot = self._format_lm_metadata_as_cot(reserve_metadata)
+ phase2_prompt = self._build_lm_prompt_with_cot(tags, lyrics, reserve_cot, is_negative_prompt=False, negative_prompt=negative_prompt)
+ phase2_prompt_negative = self._build_lm_prompt_with_cot(tags, lyrics, reserve_cot, is_negative_prompt=True, negative_prompt=negative_prompt)
+
+ prompt_len = 0
+ for text in (phase1_prompt, phase1_prompt_negative, phase2_prompt, phase2_prompt_negative):
+ try:
+ prompt_len = max(prompt_len, len(self.lm_tokenizer.encode(text)))
+ except Exception:
+ pass
+ if prompt_len <= 0:
+ prompt_len = 256
+ reserve_fn(prompt_len=prompt_len, max_tokens=reserve_max_tokens, cfg_scale=reserve_cfg)
+
def _generate_audio_codes_uncached(
self,
tags,
@@ -1002,6 +1067,17 @@ class ACEStep15Pipeline:
abort_fn=self._should_abort,
)
+ def _release_vllm_lm_runtime(self):
+ if self.lm_engine != "vllm" or self._lm_engine_impl is None:
+ return
+ release_fn = getattr(self._lm_engine_impl, "release_vram", None)
+ if not callable(release_fn):
+ return
+ try:
+ release_fn()
+ except Exception:
+ pass
+
def _generate_audio_codes(
self,
tags,
@@ -1157,7 +1233,9 @@ class ACEStep15Pipeline:
progress_unit="tokens",
)
if torch.is_tensor(cached):
+ self._release_vllm_lm_runtime()
return cached.tolist()
+ self._release_vllm_lm_runtime()
return list(cached)
def _default_timbre_latents(self, length):
@@ -1795,6 +1873,16 @@ class ACEStep15Pipeline:
user_language = None
if user_language is None and instrumental_only:
user_language = "unknown"
+ use_ref = "A" in (audio_prompt_type or "")
+ has_src_audio = bool(use_ref and audio_guide)
+ user_audio_codes = audio_codes if audio_codes is not None else audio_code_hints
+ if isinstance(user_audio_codes, str):
+ parsed = self._parse_audio_code_string(user_audio_codes)
+ user_audio_codes = parsed if parsed else None
+ elif isinstance(user_audio_codes, (list, tuple)) and user_audio_codes and isinstance(user_audio_codes[0], str):
+ parsed = self._parse_audio_code_string(user_audio_codes[0])
+ user_audio_codes = parsed if parsed else None
+ keep_vllm_lm_loaded_for_phase2 = bool(self.enable_lm and self.lm_engine == "vllm" and (not has_src_audio) and (not user_audio_codes))
phase1_metadata = {
"bpm": user_bpm,
@@ -1802,6 +1890,11 @@ class ACEStep15Pipeline:
"timesignature": user_timesignature,
"language": user_language,
}
+ if self.enable_lm:
+ try:
+ self._reserve_vllm_runtime_for_max_duration(tags, lyrics, lm_negative_prompt, lm_cfg_scale)
+ except Exception:
+ pass
tags_for_generation = tags
computed_phase1_metadata = {}
refined_caption = None
@@ -1844,6 +1937,7 @@ class ACEStep15Pipeline:
top_p=top_p,
top_k=top_k,
callback=_phase_callback,
+ release_vram_after=not keep_vllm_lm_loaded_for_phase2,
)
for field_name in missing_fields:
value = phase1_metadata.get(field_name, None) if phase1_metadata is not None else None
@@ -1900,18 +1994,7 @@ class ACEStep15Pipeline:
min_tokens = duration_int * 5
meta_cap = self._format_meta(bpm, duration_int, keyscale, timesignature)
- use_ref = "A" in (audio_prompt_type or "")
use_timbre = "B" in (audio_prompt_type or "")
- has_src_audio = bool(use_ref and audio_guide)
-
- user_audio_codes = audio_codes if audio_codes is not None else audio_code_hints
- if isinstance(user_audio_codes, str):
- parsed = self._parse_audio_code_string(user_audio_codes)
- user_audio_codes = parsed if parsed else None
- elif isinstance(user_audio_codes, (list, tuple)) and user_audio_codes:
- if isinstance(user_audio_codes[0], str):
- parsed = self._parse_audio_code_string(user_audio_codes[0])
- user_audio_codes = parsed if parsed else None
audio_codes = None
if user_audio_codes:
@@ -2110,13 +2193,3 @@ class ACEStep15Pipeline:
"audio_sampling_rate": int(self.audio_sample_rate),
"overridden_inputs": overridden_inputs,
}
-
- def close(self):
- if self._lm_engine_impl is not None:
- close_fn = getattr(self._lm_engine_impl, "close", None)
- if callable(close_fn):
- try:
- close_fn()
- except Exception:
- pass
- self._lm_engine_impl = None
diff --git a/models/TTS/ace_step15/qwen3_audio_codes.py b/models/TTS/ace_step15/qwen3_audio_codes.py
index 0ee756a..f1c209b 100644
--- a/models/TTS/ace_step15/qwen3_audio_codes.py
+++ b/models/TTS/ace_step15/qwen3_audio_codes.py
@@ -552,7 +552,9 @@ class Qwen3PtEngine:
logits_processor_update_state: Optional[Callable[[int], None]] = None,
stop_checker: Optional[Callable[[list[int], int], bool]] = None,
progress_label: str = "LM text",
+ release_vram_after: bool = True,
):
+ del release_vram_after
return generate_text_legacy(
model=self.model,
tokenizer=self.tokenizer,
@@ -582,13 +584,14 @@ class Qwen3VllmEngine:
tokenizer,
audio_code_mask: Optional[torch.Tensor],
audio_code_token_map: dict[int, int],
+ free_mmap_after_cuda_copy: bool = True,
weight_load_mode: str = "lazy",
):
self.lm_weights_path = lm_weights_path
self.tokenizer = tokenizer
self.audio_code_mask = audio_code_mask
self.audio_code_token_map = audio_code_token_map
- self.weight_load_mode = (weight_load_mode or "lazy").lower()
+ self.free_mmap_after_cuda_copy = free_mmap_after_cuda_copy
self._llm = None
self._sampling_params_cls = None
self._model_dir = None
@@ -596,6 +599,28 @@ class Qwen3VllmEngine:
self._max_num_seqs_hint = None
self._max_num_batched_tokens_hint = None
self._last_zero_code_diagnostic = None
+ self.weight_load_mode = (weight_load_mode or "lazy").strip().lower()
+ if self.weight_load_mode not in ("lazy", "pinned"):
+ raise ValueError(f"Unsupported vllm weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")
+
+ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):
+ req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(
+ prompt_len=prompt_len,
+ max_tokens=max_tokens,
+ cfg_scale=cfg_scale,
+ )
+ self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)
+
+ def _release_vram(self):
+ if self._llm is None:
+ return
+ try:
+ self._llm.unload_weights()
+ except Exception:
+ pass
+
+ def release_vram(self):
+ self._release_vram()
@staticmethod
def _compute_runtime_hints(prompt_len: int, max_tokens: int, cfg_scale: float):
@@ -675,22 +700,40 @@ class Qwen3VllmEngine:
max_num_batched_tokens=max_num_batched_tokens,
gpu_memory_utilization=gpu_memory_utilization,
tokenizer=self.tokenizer,
+ free_mmap_after_cuda_copy=self.free_mmap_after_cuda_copy,
weight_load_mode=self.weight_load_mode,
)
self._sampling_params_cls = SamplingParams
def close(self):
- if self._llm is None:
- return
- try:
- self._llm.unload_weights()
- except Exception:
- pass
- try:
- self._llm.clear_graph_cache()
- except Exception:
- pass
+ llm = getattr(self, "_llm", None)
self._llm = None
+ if llm is not None:
+ try:
+ close_fn = getattr(llm, "close", None)
+ if callable(close_fn):
+ close_fn()
+ else:
+ try:
+ llm.unload_weights()
+ except Exception:
+ pass
+ try:
+ llm.clear_graph_cache()
+ except Exception:
+ pass
+ exit_fn = getattr(llm, "exit", None)
+ if callable(exit_fn):
+ exit_fn()
+ except Exception:
+ pass
+ try:
+ del llm
+ except Exception:
+ pass
+ # Keep close() idempotent; this method is also used for runtime rebuilds.
+ self._sampling_params_cls = None
+ self._model_dir = None
def get_last_failure_reason(self) -> str:
diag = self._last_zero_code_diagnostic
@@ -888,11 +931,7 @@ class Qwen3VllmEngine:
progress_unit="tokens",
)
- if self.weight_load_mode in ("lazy", "pinned"):
- try:
- self._llm.unload_weights()
- except Exception:
- pass
+ self._release_vram()
return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)
@@ -912,6 +951,7 @@ class Qwen3VllmEngine:
logits_processor_update_state: Optional[Callable[[int], None]] = None,
stop_checker: Optional[Callable[[list[int], int], bool]] = None,
progress_label: str = "LM text",
+ release_vram_after: bool = True,
):
del stop_checker # nanovllm does not support external stop callbacks yet.
if abort_fn is not None and abort_fn():
@@ -995,11 +1035,8 @@ class Qwen3VllmEngine:
progress_unit="tokens",
)
- if self.weight_load_mode in ("lazy", "pinned"):
- try:
- self._llm.unload_weights()
- except Exception:
- pass
+ if release_vram_after:
+ self._release_vram()
return {"token_ids": token_ids, "text": text}
@@ -1012,6 +1049,7 @@ def create_qwen3_lm_engine(
lm_weights_path: str,
audio_code_mask: Optional[torch.Tensor],
audio_code_token_map: dict[int, int],
+ free_mmap_after_cuda_copy: bool = True,
weight_load_mode: str = "lazy",
):
engine = (engine_name or "legacy").strip().lower()
@@ -1023,6 +1061,7 @@ def create_qwen3_lm_engine(
tokenizer=tokenizer,
audio_code_mask=audio_code_mask,
audio_code_token_map=audio_code_token_map,
+ free_mmap_after_cuda_copy=free_mmap_after_cuda_copy,
weight_load_mode=weight_load_mode,
)
if engine == "pt":
diff --git a/models/TTS/chatterbox/mtl_tts.py b/models/TTS/chatterbox/mtl_tts.py
index ac21efc..13047a5 100644
--- a/models/TTS/chatterbox/mtl_tts.py
+++ b/models/TTS/chatterbox/mtl_tts.py
@@ -163,15 +163,15 @@ class ChatterboxMultilingualTTS:
ckpt_dir = Path(ckpt_dir)
from mmgp import offload
ve = VoiceEncoder()
- offload.load_model_data(ve, fl.locate_file( os.path.join(ckpt_folder,"ve.safetensors")))
+ offload.load_model_data(ve, fl.locate_file(os.path.join(ckpt_folder, "ve.safetensors")), writable_tensors=False)
ve.eval()
# from accelerate import init_empty_weights
# with init_empty_weights():
t3 = T3(T3Config.multilingual())
- offload.load_model_data(t3, fl.locate_file( os.path.join(ckpt_folder,"t3_mtl23ls_v2.safetensors")))
+ offload.load_model_data(t3, fl.locate_file(os.path.join(ckpt_folder, "t3_mtl23ls_v2.safetensors")), writable_tensors=False)
t3.eval()
s3gen = S3Gen()
- offload.load_model_data(s3gen, fl.locate_file( os.path.join(ckpt_folder,"s3gen.pt")))
+ offload.load_model_data(s3gen, fl.locate_file(os.path.join(ckpt_folder, "s3gen.pt")), writable_tensors=False)
s3gen.eval()
tokenizer = MTLTokenizer(
diff --git a/models/TTS/yue/pipeline.py b/models/TTS/yue/pipeline.py
index 554da84..68fd4bf 100644
--- a/models/TTS/yue/pipeline.py
+++ b/models/TTS/yue/pipeline.py
@@ -150,6 +150,7 @@ class YuePipeline:
stage1_weights_path,
modelClass=LlamaForCausalLM,
defaultConfigPath=stage1_config_path,
+ writable_tensors=False,
configKwargs={"attn_implementation": attn_implementation},
default_dtype=torch.bfloat16,
ignore_unused_weights=True,
@@ -164,6 +165,7 @@ class YuePipeline:
stage2_weights_path,
modelClass=LlamaForCausalLM,
defaultConfigPath=stage2_config_path,
+ writable_tensors=False,
configKwargs={"attn_implementation": attn_implementation},
default_dtype=torch.float16,
ignore_unused_weights=True,
diff --git a/models/flux/flux_main.py b/models/flux/flux_main.py
index 062bfc9..60b6db3 100644
--- a/models/flux/flux_main.py
+++ b/models/flux/flux_main.py
@@ -151,6 +151,7 @@ class model_factory:
fl.locate_file(os.path.join("siglip-so400m-patch14-384", "model.safetensors")),
modelClass=SiglipVisionModel,
defaultConfigPath=fl.locate_file(os.path.join("siglip-so400m-patch14-384", "vision_config.json")),
+ writable_tensors=False,
)
siglip_model.eval().to("cpu")
if len(model_filename) > 1:
@@ -162,7 +163,7 @@ class model_factory:
hidden_size=3072, #self.hidden_size,
context_layer_norm=True,
)
- offload.load_model_data(feature_embedder, model_filename[1])
+ offload.load_model_data(feature_embedder, model_filename[1], writable_tensors=False)
self.vision_encoder = siglip_model
self.vision_encoder_processor = siglip_processor
self.feature_embedder = feature_embedder
diff --git a/models/flux/modules/conditioner.py b/models/flux/modules/conditioner.py
index c37d09b..afc1d8f 100644
--- a/models/flux/modules/conditioner.py
+++ b/models/flux/modules/conditioner.py
@@ -12,13 +12,13 @@ class HFEmbedder(nn.Module):
if is_clip:
from mmgp import offload
self.tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(version, max_length=max_length)
- self.hf_module= offload.fast_load_transformers_model(os.path.join(version, "model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = os.path.join(version, "text_config.json"))
+ self.hf_module= offload.fast_load_transformers_model(os.path.join(version, "model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = os.path.join(version, "text_config.json"), writable_tensors=False)
# self.model.final_layer_norm = self.model.text_model.final_layer_norm
# self.hf_module: CLIPTextModel = CLIPTextModel.from_pretrained(version, **hf_kwargs)
else:
from mmgp import offload as offloadobj
self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(os.path.dirname(text_encoder_filename), max_length=max_length)
- self.hf_module: T5EncoderModel = offloadobj.fast_load_transformers_model(text_encoder_filename)
+ self.hf_module: T5EncoderModel = offloadobj.fast_load_transformers_model(text_encoder_filename, writable_tensors=False)
self.hf_module = self.hf_module.eval().requires_grad_(False)
diff --git a/models/flux/util.py b/models/flux/util.py
index ea28895..1e08915 100644
--- a/models/flux/util.py
+++ b/models/flux/util.py
@@ -1017,7 +1017,7 @@ def load_flow_model(
# print(f"Loading checkpoint: {ckpt_path}")
from mmgp import offload
- offload.load_model_data(model, model_filename, preprocess_sd=preprocess_sd)
+ offload.load_model_data(model, model_filename, preprocess_sd=preprocess_sd, writable_tensors=False)
# # load_sft doesn't support torch.device
# sd = load_sft(ckpt_path, device=str(device))
diff --git a/models/hyvideo/hunyuan.py b/models/hyvideo/hunyuan.py
index 2b49982..c6d26a3 100644
--- a/models/hyvideo/hunyuan.py
+++ b/models/hyvideo/hunyuan.py
@@ -259,7 +259,7 @@ class Inference(object):
from mmgp import offload
source = model_def.get("source", None)
- kwargs_light = dict(do_quantize= quantizeTransformer and not save_quantized, pinToMemory = pinToMemory, partialPinning = partialPinning)
+ kwargs_light = dict(do_quantize= quantizeTransformer and not save_quantized, pinToMemory = pinToMemory, partialPinning = partialPinning, writable_tensors=False)
if source is not None:
from wgp import save_model
offload.load_model_data(model, fl.locate_file(source), **kwargs_light)
@@ -338,7 +338,7 @@ class Inference(object):
from accelerate import init_empty_weights
with init_empty_weights():
vae = AutoencoderKLConv3D(**config)
- offload.load_model_data(vae, vae_filepath )
+ offload.load_model_data(vae, vae_filepath, writable_tensors=False)
vae = vae.to("cpu")
s_ratio = t_ratio = 1
vae._model_dtype = torch.float32 if VAE_dtype == torch.float32 else torch.float16
diff --git a/models/hyvideo/text_encoder/__init__.py b/models/hyvideo/text_encoder/__init__.py
index c770a41..9719331 100644
--- a/models/hyvideo/text_encoder/__init__.py
+++ b/models/hyvideo/text_encoder/__init__.py
@@ -192,13 +192,13 @@ class TextEncoder(nn.Module):
from mmgp import offload
if "llm" in text_encoder_type:
if "i2v" in text_encoder_type:
- self.model= offload.fast_load_transformers_model(self.model_path, modelClass= LlavaForConditionalGeneration)
+ self.model= offload.fast_load_transformers_model(self.model_path, modelClass= LlavaForConditionalGeneration, writable_tensors=False)
else:
- self.model= offload.fast_load_transformers_model(self.model_path, modelPrefix="language_model", forcedConfigPath = fl.locate_file("llava-llama-3-8b/config.json"))
+ self.model= offload.fast_load_transformers_model(self.model_path, modelPrefix="language_model", forcedConfigPath = fl.locate_file("llava-llama-3-8b/config.json"), writable_tensors=False)
self.model.final_layer_norm = self.model.model.norm
else:
- self.model= offload.fast_load_transformers_model(fl.locate_file("clip_vit_large_patch14/model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = fl.locate_file("clip_vit_large_patch14/text_config.json"))
+ self.model= offload.fast_load_transformers_model(fl.locate_file("clip_vit_large_patch14/model.safetensors"), ignore_unused_weights= True, modelClass=CLIPTextModel, forcedConfigPath = fl.locate_file("clip_vit_large_patch14/text_config.json"), writable_tensors=False)
self.model.final_layer_norm = self.model.text_model.final_layer_norm
self.dtype = self.model.dtype
diff --git a/models/hyvideo/vae/__init__.py b/models/hyvideo/vae/__init__.py
index 73c0032..6d2b422 100644
--- a/models/hyvideo/vae/__init__.py
+++ b/models/hyvideo/vae/__init__.py
@@ -54,7 +54,7 @@ def load_vae(vae_type: str="884-16c-hy",
# offload.save_model(vae, "vae_32.safetensors")
# vae.to(torch.bfloat16)
# offload.save_model(vae, "vae_16.safetensors")
- offload.load_model_data(vae, vae_path )
+ offload.load_model_data(vae, vae_path, writable_tensors=False)
# ckpt = torch.load(vae_ckpt, weights_only=True, map_location=vae.device)
spatial_compression_ratio = vae.config.spatial_compression_ratio
diff --git a/models/kandinsky5/kandinsky/models/text_embedders.py b/models/kandinsky5/kandinsky/models/text_embedders.py
index 42d183e..075fff6 100644
--- a/models/kandinsky5/kandinsky/models/text_embedders.py
+++ b/models/kandinsky5/kandinsky/models/text_embedders.py
@@ -16,7 +16,7 @@ from shared.utils.text_encoder_cache import TextEncoderCache
class ClipTextEmbedder:
def __init__(self, conf, device):
- self.model= offload.fast_load_transformers_model( os.path.join(conf.checkpoint_path,"model.safetensors"), modelClass=CLIPTextModel, ignore_unused_weights= True, forcedConfigPath = os.path.join(conf.checkpoint_path, "text_config.json"))
+ self.model= offload.fast_load_transformers_model(os.path.join(conf.checkpoint_path, "model.safetensors"), modelClass=CLIPTextModel, ignore_unused_weights= True, forcedConfigPath=os.path.join(conf.checkpoint_path, "text_config.json"), writable_tensors=False)
self.model.final_layer_norm = self.model.text_model.final_layer_norm
self.model.eval()
# self.model = freeze(self.model)
diff --git a/models/longcat/longcat_main.py b/models/longcat/longcat_main.py
index cd2c762..aa32c13 100644
--- a/models/longcat/longcat_main.py
+++ b/models/longcat/longcat_main.py
@@ -147,6 +147,7 @@ class LongCatModel:
vae_weights,
modelClass=AutoencoderKLWan,
defaultConfigPath=fl.locate_file(vae_cfg_path),
+ writable_tensors=False,
preprocess_sd=preprocess_vae_sd,
default_dtype=VAE_dtype,
)
diff --git a/models/ltx2/ltx2.py b/models/ltx2/ltx2.py
index 93a83ee..cc116d1 100644
--- a/models/ltx2/ltx2.py
+++ b/models/ltx2/ltx2.py
@@ -499,6 +499,7 @@ class LTX2:
path,
postprocess_sd=postprocess,
default_dtype=self.dtype,
+ writable_tensors=False,
ignore_missing_keys=False,
)
model.eval().requires_grad_(False)
diff --git a/models/ltx2/ltx_core/loader/single_gpu_model_builder.py b/models/ltx2/ltx_core/loader/single_gpu_model_builder.py
index aa2d136..c276342 100644
--- a/models/ltx2/ltx_core/loader/single_gpu_model_builder.py
+++ b/models/ltx2/ltx_core/loader/single_gpu_model_builder.py
@@ -141,6 +141,7 @@ class SingleGPUModelBuilder(Generic[ModelType], ModelBuilderProtocol[ModelType],
meta_model,
[(sd, quantization_map)],
default_dtype=dtype or torch.bfloat16,
+ writable_tensors=False,
ignore_missing_keys=self.ignore_missing_keys,
)
return self._return_model(meta_model, device)
@@ -163,6 +164,7 @@ class SingleGPUModelBuilder(Generic[ModelType], ModelBuilderProtocol[ModelType],
meta_model,
model_paths,
default_dtype=dtype or torch.bfloat16,
+ writable_tensors=False,
preprocess_sd=preprocess_sd,
ignore_missing_keys=self.ignore_missing_keys,
)
diff --git a/models/ltx_video/ltxv.py b/models/ltx_video/ltxv.py
index 8038987..5898576 100644
--- a/models/ltx_video/ltxv.py
+++ b/models/ltx_video/ltxv.py
@@ -178,7 +178,7 @@ class LTXV:
# offload.save_model(transformer, "ckpts/ltxv_0.9.8_13B_quanto_bf16_int8.safetensors", do_quantize= True, config_file_path= "c:/temp/ltxv_config.json")
# vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)
- vae = offload.fast_load_transformers_model( fl.locate_file("ltxv_0.9.7_VAE.safetensors"), modelClass=CausalVideoAutoencoder)
+ vae = offload.fast_load_transformers_model(fl.locate_file("ltxv_0.9.7_VAE.safetensors"), modelClass=CausalVideoAutoencoder, writable_tensors=False)
# vae = offload.fast_load_transformers_model("ckpts/ltxv_0.9.8_VAE.safetensors", modelClass=CausalVideoAutoencoder)
# if VAE_dtype == torch.float16:
VAE_dtype = torch.bfloat16
@@ -188,7 +188,7 @@ class LTXV:
# offload.save_model(vae, "vae.safetensors", config_file_path="c:/temp/config_vae.json")
# model_filepath = "c:/temp/ltxd/ltxv-13b-0.9.7-distilled.safetensors"
- transformer = offload.fast_load_transformers_model(model_filepath, modelClass=Transformer3DModel)
+ transformer = offload.fast_load_transformers_model(model_filepath, modelClass=Transformer3DModel, writable_tensors=False)
# offload.save_model(transformer, "ckpts/ltxv_0.9.7_13B_distilled_bf16.safetensors", config_file_path= "c:/temp/ltxd/config.json")
# offload.save_model(transformer, "ckpts/ltxv_0.9.7_13B_distilled_quanto_bf16_int8.safetensors", do_quantize= True, config_file_path="c:/temp/ltxd/config.json")
# transformer = offload.fast_load_transformers_model(model_filepath, modelClass=Transformer3DModel)
@@ -215,7 +215,7 @@ class LTXV:
# offload.save_model(text_encoder, "T5_xxl_1.1_enc_bf16.safetensors", config_file_path="T5_config.json")
# offload.save_model(text_encoder, "T5_xxl_1.1_enc_quanto_bf16_int8.safetensors", do_quantize= True, config_file_path="T5_config.json")
- text_encoder = offload.fast_load_transformers_model(text_encoder_filepath)
+ text_encoder = offload.fast_load_transformers_model(text_encoder_filepath, writable_tensors=False)
patchifier = SymmetricPatchifier(patch_size=1)
text_encoder_folder = self.model_def.get("text_encoder_folder")
if text_encoder_folder:
diff --git a/models/qwen/qwen_main.py b/models/qwen/qwen_main.py
index 808d225..e9c6498 100644
--- a/models/qwen/qwen_main.py
+++ b/models/qwen/qwen_main.py
@@ -87,9 +87,9 @@ class model_factory():
source = model_def.get("source", None)
if source is not None:
- offload.load_model_data(transformer, source, fused_split_map=_QWEN_FUSED_SPLIT_MAP)
+ offload.load_model_data(transformer, source, fused_split_map=_QWEN_FUSED_SPLIT_MAP, writable_tensors=False)
else:
- offload.load_model_data(transformer, transformer_filename, fused_split_map=_QWEN_FUSED_SPLIT_MAP)
+ offload.load_model_data(transformer, transformer_filename, fused_split_map=_QWEN_FUSED_SPLIT_MAP, writable_tensors=False)
# transformer = offload.fast_load_transformers_model("transformer_quanto.safetensors", writable_tensors= True , modelClass=QwenImageTransformer2DModel, defaultConfigPath="transformer_config.json")
if not source is None:
diff --git a/models/wan/any2video.py b/models/wan/any2video.py
index df985d1..e886008 100644
--- a/models/wan/any2video.py
+++ b/models/wan/any2video.py
@@ -953,7 +953,7 @@ class WanAny2V:
if True:
with init_empty_weights():
arc_resampler = Resampler( depth=4, dim=1280, dim_head=64, embedding_dim=512, ff_mult=4, heads=20, num_queries=16, output_dim=2048 if lynx_lite else 5120 )
- offload.load_model_data(arc_resampler, fl.locate_file("wan2.1_lynx_lite_arc_resampler.safetensors" if lynx_lite else "wan2.1_lynx_full_arc_resampler.safetensors"))
+ offload.load_model_data(arc_resampler, fl.locate_file("wan2.1_lynx_lite_arc_resampler.safetensors" if lynx_lite else "wan2.1_lynx_full_arc_resampler.safetensors"), writable_tensors=False)
arc_resampler.to(self.device)
arcface_embed = face_arc_embeds[None,None,:].to(device=self.device, dtype=torch.float)
ip_hidden_states = arc_resampler(arcface_embed).to(self.dtype)
diff --git a/models/wan/fantasytalking/infer.py b/models/wan/fantasytalking/infer.py
index a756417..8bb0917 100644
--- a/models/wan/fantasytalking/infer.py
+++ b/models/wan/fantasytalking/infer.py
@@ -17,7 +17,7 @@ def parse_audio(audio_path, start_frame, num_frames, fps = 23, device = "cuda"):
with init_empty_weights():
proj_model = AudioProjModel( 768, 2048)
- offload.load_model_data(proj_model, fl.locate_file("fantasy_proj_model.safetensors"))
+ offload.load_model_data(proj_model, fl.locate_file("fantasy_proj_model.safetensors"), writable_tensors=False)
proj_model.to("cpu").eval().requires_grad_(False)
wav2vec_model_dir = fl.locate_folder("wav2vec")
@@ -34,4 +34,4 @@ def parse_audio(audio_path, start_frame, num_frames, fps = 23, device = "cuda"):
gc.collect()
torch.cuda.empty_cache()
- return audio_proj_split, audio_context_lens
\ No newline at end of file
+ return audio_proj_split, audio_context_lens
diff --git a/models/wan/multitalk/multitalk.py b/models/wan/multitalk/multitalk.py
index ca732c3..f0a65ce 100644
--- a/models/wan/multitalk/multitalk.py
+++ b/models/wan/multitalk/multitalk.py
@@ -28,6 +28,7 @@ def custom_init(device, wav2vec):
os.path.join(wav2vec_folder, "pytorch_model.bin"),
modelClass=Wav2Vec2Model,
defaultConfigPath=os.path.join(wav2vec_folder, "config.json"),
+ writable_tensors=False,
).to(device)
audio_encoder.feature_extractor._freeze_parameters()
wav2vec_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
diff --git a/models/wan/ovi_fusion_engine.py b/models/wan/ovi_fusion_engine.py
index 0474231..437c4f1 100644
--- a/models/wan/ovi_fusion_engine.py
+++ b/models/wan/ovi_fusion_engine.py
@@ -71,8 +71,8 @@ class OviFusionEngine:
self.target_dtype = torch.bfloat16 # dtype, wont work with torch.float16
model, video_config, audio_config = init_fusion_score_model_ovi()
# offload.load_model_data(model, "c:/temp/model_960x960.safetensors")
- offload.load_model_data(model.video_model, model_filename[0])
- offload.load_model_data(model.audio_model, model_filename[1])
+ offload.load_model_data(model.video_model, model_filename[0], writable_tensors=False)
+ offload.load_model_data(model.audio_model, model_filename[1], writable_tensors=False)
offload.change_dtype(model, dtype, True)
model = model.eval()
# model.set_rope_params()
diff --git a/preprocessing/matanyone/tools/base_segmenter.py b/preprocessing/matanyone/tools/base_segmenter.py
index 09fa309..5327037 100644
--- a/preprocessing/matanyone/tools/base_segmenter.py
+++ b/preprocessing/matanyone/tools/base_segmenter.py
@@ -139,7 +139,7 @@ class BaseSegmenter:
# self.model.to(torch.float16)
# offload.save_model(self.model, "ckpts/mask/sam_vit_h_4b8939_fp16.safetensors")
- offload.load_model_data(self.model, fl.locate_file("mask/sam_vit_h_4b8939_fp16.safetensors"))
+ offload.load_model_data(self.model, fl.locate_file("mask/sam_vit_h_4b8939_fp16.safetensors"), writable_tensors=False)
self.model.to(torch.float32) # need to be optimized, if not f32 crappy precision
self.model.to(device=self.device)
self.predictor = SamPredictor(self.model)
diff --git a/shared/llm_engines/nanovllm/config.py b/shared/llm_engines/nanovllm/config.py
index 69513e2..c39c3ea 100644
--- a/shared/llm_engines/nanovllm/config.py
+++ b/shared/llm_engines/nanovllm/config.py
@@ -12,8 +12,9 @@ class Config:
gpu_memory_utilization: float = 0.9
tensor_parallel_size: int = 1
enforce_eager: bool = False
- weight_load_mode: str = "eager" # eager | lazy | pinned
+ weight_load_mode: str = "lazy" # lazy | pinned
hf_config: AutoConfig | None = None
+ free_mmap_after_cuda_copy: bool = True
eos: int = -1
kvcache_block_size: int = 256
num_kvcache_blocks: int = -1
@@ -29,6 +30,8 @@ class Config:
self.model_dir = self.model
assert self.kvcache_block_size % 256 == 0
assert 1 <= self.tensor_parallel_size <= 8
+ if self.weight_load_mode not in ("lazy", "pinned"):
+ raise ValueError(f"Unsupported weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")
self.hf_config = AutoConfig.from_pretrained(self.model_dir)
self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings)
assert self.max_num_batched_tokens >= self.max_model_len
diff --git a/shared/llm_engines/nanovllm/engine/llm_engine.py b/shared/llm_engines/nanovllm/engine/llm_engine.py
index b48d179..6edec65 100644
--- a/shared/llm_engines/nanovllm/engine/llm_engine.py
+++ b/shared/llm_engines/nanovllm/engine/llm_engine.py
@@ -4,7 +4,6 @@ from time import perf_counter
from tqdm.auto import tqdm
from transformers import AutoTokenizer
import torch.multiprocessing as mp
-import torch
from nanovllm.config import Config
from nanovllm.sampling_params import SamplingParams
@@ -38,22 +37,80 @@ class LLMEngine:
self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True)
config.eos = self.tokenizer.eos_token_id
self.scheduler = Scheduler(config)
- atexit.register(self.exit)
+ self._exit_registered = False
+ self._closed = False
+ self._exited = False
+ self._atexit_callback = self.exit
+ atexit.register(self._atexit_callback)
+ self._exit_registered = True
def exit(self):
- self.model_runner.call("exit")
- del self.model_runner
- for p in self.ps:
- p.join()
+ if self._exited:
+ return
+ self._exited = True
+ runner = getattr(self, "model_runner", None)
+ if runner is not None:
+ try:
+ runner.call("exit")
+ except Exception:
+ pass
+ try:
+ del self.model_runner
+ except Exception:
+ pass
+ for p in list(getattr(self, "ps", [])):
+ try:
+ p.join()
+ except Exception:
+ pass
+ self.ps = []
+ self.events = []
+
+ def close(self):
+ if self._closed:
+ return
+ self._closed = True
+ try:
+ self.unload_weights()
+ except Exception:
+ pass
+ try:
+ self.clear_graph_cache()
+ except Exception:
+ pass
+ try:
+ self.exit()
+ except Exception:
+ pass
+ if self._exit_registered:
+ try:
+ atexit.unregister(self._atexit_callback)
+ except Exception:
+ pass
+ self._exit_registered = False
+ self._atexit_callback = None
+ self.scheduler = None
+ self.tokenizer = None
+
+ def __del__(self):
+ try:
+ self.close()
+ except Exception:
+ pass
def unload_weights(self):
- self.model_runner.unload_weights()
+ runner = getattr(self, "model_runner", None)
+ if runner is None:
+ return
+ runner.unload_weights()
# KV cache is invalid after unload/reload, so cached prefix block metadata
# must be dropped as well to prevent stale-cache reuse.
try:
self.reset()
except Exception:
pass
+ if self.scheduler is None:
+ return
self.scheduler.waiting.clear()
self.scheduler.running.clear()
self.scheduler.block_manager = BlockManager(
@@ -62,13 +119,22 @@ class LLMEngine:
)
def clear_graph_cache(self):
- self.model_runner.clear_graph_cache()
+ runner = getattr(self, "model_runner", None)
+ if runner is None:
+ return
+ runner.clear_graph_cache()
def reset_guard_counts(self):
- self.model_runner.call("reset_guard_counts")
+ runner = getattr(self, "model_runner", None)
+ if runner is None:
+ return
+ runner.call("reset_guard_counts")
def get_guard_counts(self, reset: bool = False):
- return self.model_runner.call("get_guard_counts", reset)
+ runner = getattr(self, "model_runner", None)
+ if runner is None:
+ return {}
+ return runner.call("get_guard_counts", reset)
def add_request(self, prompt: str | list[int], sampling_params: SamplingParams, unconditional_prompt: str | list[int] | None = None):
if isinstance(prompt, str):
@@ -109,6 +175,8 @@ class LLMEngine:
return outputs, num_tokens
def is_finished(self):
+ if self.scheduler is None:
+ return True
return self.scheduler.is_finished()
def reset(self):
@@ -118,6 +186,8 @@ class LLMEngine:
KV cache block leaks that can cause 'deque index out of range' errors.
"""
# Deallocate all running sequences
+ if self.scheduler is None:
+ return
while self.scheduler.running:
seq = self.scheduler.running.popleft()
if seq.block_table: # Only deallocate if blocks are allocated
@@ -136,6 +206,8 @@ class LLMEngine:
use_tqdm: bool = True,
unconditional_prompts: list[str] | list[list[int]] | None = None,
) -> list[str]:
+ if self.scheduler is None:
+ raise RuntimeError("LLM engine is closed.")
# Ensure weights/KV cache are ready for lazy/pinned modes, and sync scheduler blocks.
self.model_runner.ensure_weights_loaded()
if (self.config.num_kvcache_blocks > 0 and
@@ -163,10 +235,7 @@ class LLMEngine:
break
except Exception:
seed_to_apply = None
- if seed_to_apply is not None:
- torch.manual_seed(seed_to_apply)
- if torch.cuda.is_available():
- torch.cuda.manual_seed(seed_to_apply)
+ self.model_runner.call("set_sampling_seed", seed_to_apply)
if unconditional_prompts is None:
unconditional_prompts = [None] * len(prompts)
for prompt, sp, uncond_prompt in zip(prompts, sampling_params, unconditional_prompts):
diff --git a/shared/llm_engines/nanovllm/engine/model_runner.py b/shared/llm_engines/nanovllm/engine/model_runner.py
index f9015a5..9b765ee 100644
--- a/shared/llm_engines/nanovllm/engine/model_runner.py
+++ b/shared/llm_engines/nanovllm/engine/model_runner.py
@@ -1,4 +1,5 @@
import pickle
+import gc
import torch
import torch.distributed as dist
from multiprocessing.synchronize import Event
@@ -7,7 +8,7 @@ import sys
from nanovllm.config import Config
from nanovllm.engine.sequence import Sequence
-from nanovllm.models.qwen3 import Qwen3ForCausalLM
+from nanovllm.models import resolve_model_class
from nanovllm.layers.sampler import Sampler
from nanovllm.utils.context import set_context, get_context, reset_context
from nanovllm.utils.loader import load_model, WeightStore
@@ -90,39 +91,26 @@ class ModelRunner:
config_dtype = torch.bfloat16
self.dtype = config_dtype # Save for later use
- self.weight_load_mode = (config.weight_load_mode or "eager").lower()
+ self.weight_load_mode = (config.weight_load_mode or "lazy").lower()
self._weights_loaded = False
self._weight_store = None
self._is_quanto_int8 = False
+ self._model_cls = resolve_model_class(hf_config)
+ self._model_is_meta = False
self._graph_cache = {}
self._graph_cache_order = []
self._logits_bias_cache = {}
+ self._sampling_generator = None
self._guard_counts = {}
self._guard_seen_details = set()
torch.set_default_dtype(config_dtype)
- if self.weight_load_mode in ("lazy", "pinned"):
- torch.set_default_device("cpu")
- self.model = Qwen3ForCausalLM(hf_config)
- self._weight_store = WeightStore(config.model_file or config.model_dir, mode=self.weight_load_mode)
- self._is_quanto_int8 = bool(getattr(self._weight_store, "is_quanto_int8", False))
- else:
- torch.set_default_device("cuda")
- self.model = Qwen3ForCausalLM(hf_config)
- load_model(self.model, config.model_file or config.model_dir)
- self._retie_word_embeddings_if_needed()
- self._weights_loaded = True
+ self.model = self._build_deferred_model(hf_config)
self.sampler = Sampler()
# Pre-allocate buffers for sampling (optimization: avoid repeated tensor creation)
- # Must be called before warmup_model() since it uses these buffers
+ # Must be called before model execution paths that use these buffers.
self._allocate_sample_buffers()
- if self._weights_loaded:
- self.warmup_model()
- self.allocate_kv_cache()
- if not self.enforce_eager:
- self.capture_cudagraph()
-
torch.set_default_device("cpu")
torch.set_default_dtype(default_dtype)
@@ -135,59 +123,125 @@ class ModelRunner:
self.shm = SharedMemory(name="nanovllm")
self.loop()
+ def _build_deferred_model(self, hf_config):
+ # Lazy/pinned mode should not allocate full CPU parameter storage upfront.
+ with torch.device("meta"):
+ model = self._model_cls(hf_config)
+ self._model_is_meta = True
+ return model
+
def ensure_weights_loaded(self):
if self._weights_loaded:
return
+ print(f"Loading LM Weights '{self.config.model}' in vllm...")
default_dtype = torch.get_default_dtype()
torch.set_default_dtype(self.dtype)
- torch.set_default_device("cuda")
- if self._is_quanto_int8:
- for module in self.model.modules():
- prepare = getattr(module, "prepare_for_quantized_load", None)
- if callable(prepare):
- prepare()
- self.model = self.model.to("cuda")
- load_model(self.model, "", weight_store=self._weight_store)
- self._retie_word_embeddings_if_needed()
- self._weights_loaded = True
- self.warmup_model()
- self.allocate_kv_cache()
- if not self.enforce_eager:
- self.capture_cudagraph()
- torch.set_default_device("cpu")
- torch.set_default_dtype(default_dtype)
-
- def _retie_word_embeddings_if_needed(self):
- # Some quantized checkpoints omit lm_head.weight and rely on tied embeddings.
- # After device moves/load cycles, the tie can be broken; restore it explicitly.
try:
- lm_head = getattr(self.model, "lm_head", None)
- embed = getattr(self.model, "embed_tokens", None)
- if lm_head is None or embed is None:
- return
- lm_w = getattr(lm_head, "weight", None)
- emb_w = getattr(embed, "weight", None)
- if lm_w is None or emb_w is None:
- return
- if lm_w.shape != emb_w.shape:
- return
- if lm_w.data_ptr() != emb_w.data_ptr():
- lm_head.weight.data = emb_w.data
- except Exception:
+ self._ensure_weight_store()
+ if self.model is None:
+ self.model = self._build_deferred_model(self.config.hf_config)
+ self._tie_word_embeddings_if_needed()
+ self._mark_tied_meta_parameter_ids()
+ # Tie before materialization so lm_head does not allocate a duplicate tensor.
+ was_meta_model = bool(self._model_is_meta)
+ if self._is_quanto_int8:
+ for module in self.model.modules():
+ prepare = getattr(module, "prepare_for_quantized_load", None)
+ if callable(prepare):
+ prepare()
+ # Keep model meta until each parameter is replaced directly from safetensor
+ # to avoid allocating a dense, fully materialized CUDA shell first.
+ load_model(
+ self.model,
+ "",
+ weight_store=self._weight_store,
+ # We keep tensors tied to the active weight reader only within each
+ # load iteration; explicit cloning is not needed and avoids a full
+ # model-sized temporary copy in RAM.
+ clone_loaded_tensors=False,
+ )
+ if was_meta_model:
+ self._materialize_runtime_buffers()
+ self._model_is_meta = False
+ self._close_weight_store_if_lazy()
+ self._weights_loaded = True
+ self.allocate_kv_cache()
+ if not self.enforce_eager:
+ self.capture_cudagraph()
+ finally:
+ if not self._weights_loaded:
+ self._close_weight_store_if_lazy()
+ torch.set_default_device("cpu")
+ torch.set_default_dtype(default_dtype)
+
+ def _ensure_weight_store(self):
+ if self._weight_store is not None:
+ return
+ self._weight_store = WeightStore(self.config.model_file or self.config.model_dir, mode=self.weight_load_mode)
+ self._is_quanto_int8 = bool(getattr(self._weight_store, "is_quanto_int8", False))
+
+ def _close_weight_store_if_lazy(self):
+ if self._weight_store is None:
+ return
+ close_fn = getattr(self._weight_store, "close", None)
+ if callable(close_fn):
+ close_fn()
+ self._weight_store = None
+
+ def _materialize_runtime_buffers(self):
+ if self.model is None:
+ return
+ for module in self.model.modules():
+ materialize = getattr(module, "materialize_cache", None)
+ if callable(materialize):
+ materialize()
+
+ def _get_tied_embeddings(self):
+ if self.model is None:
+ return None
+ hf_config = getattr(self.config, "hf_config", None)
+ if not bool(getattr(hf_config, "tie_word_embeddings", False)):
+ return None
+ lm_head = getattr(self.model, "lm_head", None)
+ embed = getattr(self.model, "embed_tokens", None)
+ if lm_head is None or embed is None:
+ return None
+ lm_w = getattr(lm_head, "weight", None)
+ emb_w = getattr(embed, "weight", None)
+ if lm_w is None or emb_w is None:
+ return None
+ return lm_head, embed
+
+ def _mark_tied_meta_parameter_ids(self):
+ tied = self._get_tied_embeddings()
+ if tied is None:
+ self.model._vllm_tied_param_ids = set()
+ return
+ lm_head, embed = tied
+ if lm_head.weight is not embed.weight:
+ self.model._vllm_tied_param_ids = set()
+ return
+ self.model._vllm_tied_param_ids = {id(embed.weight)}
+
+ def _tie_word_embeddings_if_needed(self):
+ tied = self._get_tied_embeddings()
+ if tied is None:
return
+ lm_head, embed = tied
+ if lm_head.weight is not embed.weight:
+ lm_head.register_parameter("weight", embed.weight)
+ if lm_head.weight is not embed.weight:
+ raise RuntimeError("Failed to retie lm_head.weight with embed_tokens.weight.")
def unload_weights(self):
if not self._weights_loaded:
return
- try:
- self.model = self.model.to("cpu")
- except Exception:
- pass
# Clear attention KV cache refs so we don't write into freed storage later.
try:
- for module in self.model.modules():
- if hasattr(module, "k_cache") and hasattr(module, "v_cache"):
- module.k_cache = module.v_cache = torch.tensor([])
+ if self.model is not None:
+ for module in self.model.modules():
+ if hasattr(module, "k_cache") and hasattr(module, "v_cache"):
+ module.k_cache = module.v_cache = torch.tensor([])
except Exception:
pass
if hasattr(self, "kv_cache"):
@@ -213,7 +267,19 @@ class ModelRunner:
except Exception:
pass
self._logits_bias_cache.clear()
+ self._sampling_generator = None
self._weights_loaded = False
+ self.model = None
+ self._model_is_meta = False
+ if self._weight_store is not None:
+ try:
+ close_fn = getattr(self._weight_store, "close", None)
+ if callable(close_fn):
+ close_fn()
+ except Exception:
+ pass
+ self._weight_store = None
+ gc.collect()
def _get_graph_capture_signature(self):
model_ptr = -1
@@ -231,6 +297,12 @@ class ModelRunner:
pass
return (model_ptr, kv_ptr, int(self.config.max_model_len), int(self.config.max_num_seqs))
+ def _get_model_device(self) -> torch.device:
+ try:
+ return next(self.model.parameters()).device
+ except Exception:
+ return torch.device("cpu")
+
def _drop_graph_cache_entry(self, cache_key):
entry = self._graph_cache.pop(cache_key, None)
if cache_key in self._graph_cache_order:
@@ -286,6 +358,14 @@ class ModelRunner:
self._logits_bias_cache[key] = cached
return cached
+ def set_sampling_seed(self, seed: int | None):
+ if seed is None or not torch.cuda.is_available():
+ self._sampling_generator = None
+ return
+ generator = torch.Generator(device="cuda")
+ generator.manual_seed(int(seed))
+ self._sampling_generator = generator
+
@staticmethod
def _apply_logits_bias(logits_row: torch.Tensor, bias: torch.Tensor):
logits_row.add_(bias)
@@ -323,7 +403,53 @@ class ModelRunner:
# Max length is max_model_len since sequences can be that long
self._seq_token_ids_buffer = torch.zeros(max_bs, self.config.max_model_len, dtype=torch.int64, device="cpu", pin_memory=True)
+ def _release_sample_buffers(self):
+ buffer_names = [
+ "_cpu_temperatures",
+ "_cpu_cfg_scales",
+ "_cpu_top_ks",
+ "_cpu_top_ps",
+ "_cpu_repetition_penalties",
+ "_cpu_input_ids",
+ "_cpu_positions",
+ "_cpu_slot_mapping",
+ "_cpu_context_lens",
+ "_cpu_prefill_input_ids",
+ "_cpu_prefill_positions",
+ "_cpu_prefill_cu_seqlens",
+ "_cpu_prefill_slot_mapping",
+ "_cpu_block_tables",
+ "_seq_token_ids_buffer",
+ ]
+ for name in buffer_names:
+ if hasattr(self, name):
+ try:
+ delattr(self, name)
+ except Exception:
+ pass
+
def exit(self):
+ try:
+ self.unload_weights()
+ except Exception:
+ pass
+ try:
+ if self._weight_store is not None:
+ close_fn = getattr(self._weight_store, "close", None)
+ if callable(close_fn):
+ close_fn()
+ except Exception:
+ pass
+ self._weight_store = None
+ self._release_sample_buffers()
+ self._logits_bias_cache.clear()
+ self._guard_counts.clear()
+ self._guard_seen_details.clear()
+ if hasattr(self, "sampler"):
+ self.sampler = None
+ if hasattr(self, "model"):
+ self.model = None
+ self._model_is_meta = False
if self.world_size > 1:
self.shm.close()
dist.barrier()
@@ -338,6 +464,10 @@ class ModelRunner:
torch.cuda.synchronize()
except Exception:
pass
+ try:
+ torch.cuda.empty_cache()
+ except Exception:
+ pass
if dist.is_initialized():
dist.destroy_process_group()
@@ -371,58 +501,51 @@ class ModelRunner:
method = getattr(self, method_name, None)
return method(*args)
- def warmup_model(self):
- torch.cuda.empty_cache()
- torch.cuda.reset_peak_memory_stats()
- max_num_batched_tokens, max_model_len = self.config.max_num_batched_tokens, self.config.max_model_len
- num_seqs = min(max_num_batched_tokens // max_model_len, self.config.max_num_seqs)
- seqs = [Sequence([0] * max_model_len) for _ in range(num_seqs)]
- self.run(seqs, True)
- torch.cuda.empty_cache()
-
def allocate_kv_cache(self):
config = self.config
hf_config = config.hf_config
- free, total = torch.cuda.mem_get_info()
- current = torch.cuda.memory_stats()["allocated_bytes.all.current"]
num_kv_heads = hf_config.num_key_value_heads // self.world_size
head_dim = getattr(hf_config, "head_dim", hf_config.hidden_size // hf_config.num_attention_heads)
block_bytes = 2 * hf_config.num_hidden_layers * self.block_size * num_kv_heads * head_dim * self.dtype.itemsize
-
- # Calculate available memory for KV cache
- # After warmup_model, empty_cache has been called, so current represents model memory only
- # Use free memory but respect the gpu_memory_utilization limit
- target_total_usage = total * config.gpu_memory_utilization
- available_for_kv_cache = min(free * 0.9, target_total_usage - current)
-
- # Ensure we have positive memory available
- if available_for_kv_cache <= 0:
- available_for_kv_cache = free * 0.5 # Fallback to 50% of free memory
-
- config.num_kvcache_blocks = max(1, int(available_for_kv_cache) // block_bytes)
- # Cap KV cache blocks to what is required by max_model_len and max_num_seqs.
- # This keeps VRAM usage proportional to the requested token budget (incl. CFG).
+
+ # Strict policy: allocate exactly the blocks required by requested runtime limits.
required_blocks_per_seq = (config.max_model_len + self.block_size - 1) // self.block_size
required_total_blocks = required_blocks_per_seq * max(1, config.max_num_seqs)
- if required_total_blocks > 0:
- config.num_kvcache_blocks = min(config.num_kvcache_blocks, required_total_blocks)
- if config.num_kvcache_blocks <= 0:
+ config.num_kvcache_blocks = max(1, int(required_total_blocks))
+ required_kv_bytes = config.num_kvcache_blocks * block_bytes
+ free, total = torch.cuda.mem_get_info()
+ current = torch.cuda.memory_stats()["allocated_bytes.all.current"]
+ target_total_usage = total * config.gpu_memory_utilization
+ allowed_kv_bytes = max(0, target_total_usage - current)
+ if required_kv_bytes > allowed_kv_bytes:
raise RuntimeError(
- f"Insufficient GPU memory for KV cache. "
+ f"Insufficient GPU memory for strict KV cache sizing under gpu_memory_utilization={config.gpu_memory_utilization:.2f}. "
+ f"Required KV: {required_kv_bytes / 1024**3:.2f} GB "
+ f"(blocks={config.num_kvcache_blocks}, block={block_bytes / 1024**2:.2f} MB), "
+ f"Allowed by limit: {allowed_kv_bytes / 1024**3:.2f} GB, "
f"Free: {free / 1024**3:.2f} GB, Current: {current / 1024**3:.2f} GB, "
- f"Available for KV: {available_for_kv_cache / 1024**3:.2f} GB, "
- f"Block size: {block_bytes / 1024**2:.2f} MB"
+ f"Requested max_model_len={config.max_model_len}, max_num_seqs={config.max_num_seqs}."
)
- self.kv_cache = torch.empty(
- 2,
- hf_config.num_hidden_layers,
- config.num_kvcache_blocks,
- self.block_size,
- num_kv_heads,
- head_dim,
- device="cuda",
- dtype=self.dtype,
- )
+ try:
+ self.kv_cache = torch.empty(
+ 2,
+ hf_config.num_hidden_layers,
+ config.num_kvcache_blocks,
+ self.block_size,
+ num_kv_heads,
+ head_dim,
+ device="cuda",
+ dtype=self.dtype,
+ )
+ except RuntimeError as exc:
+ if "out of memory" in str(exc).lower():
+ free_now, total_now = torch.cuda.mem_get_info()
+ raise RuntimeError(
+ f"Failed to allocate strict KV cache ({required_kv_bytes / 1024**3:.2f} GB) for "
+ f"max_model_len={config.max_model_len}, max_num_seqs={config.max_num_seqs}. "
+ f"Current free VRAM: {free_now / 1024**3:.2f} GB / {total_now / 1024**3:.2f} GB total."
+ ) from exc
+ raise
layer_id = 0
for module in self.model.modules():
if hasattr(module, "k_cache") and hasattr(module, "v_cache"):
@@ -663,6 +786,7 @@ class ModelRunner:
top_ks=top_ks if top_ks is not None else None,
top_ps=top_ps if top_ps is not None else None,
repetition_penalties=None, # Already applied above
+ generator=self._sampling_generator,
# input_ids=cond_input_ids,
).tolist()
@@ -735,6 +859,7 @@ class ModelRunner:
top_ks=top_ks if top_ks is not None else None,
top_ps=top_ps if top_ps is not None else None,
repetition_penalties=None, # Already applied above
+ generator=self._sampling_generator,
# input_ids=seq_input_ids,
).tolist()
@@ -753,6 +878,7 @@ class ModelRunner:
def capture_cudagraph(self):
config = self.config
cache_key = (config.max_model_len, config.max_num_seqs)
+ model_device = self._get_model_device()
cached = self._graph_cache.get(cache_key)
if cached is not None:
current_sig = self._get_graph_capture_signature()
@@ -765,16 +891,16 @@ class ModelRunner:
self._graph_cache_order.remove(cache_key)
self._graph_cache_order.append(cache_key)
return
- self._drop_graph_cache_entry(cache_key)
+ self._drop_graph_cache_entry(cache_key)
hf_config = config.hf_config
max_bs = min(self.config.max_num_seqs, 512)
max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size
- input_ids = torch.zeros(max_bs, dtype=torch.int64)
- positions = torch.zeros(max_bs, dtype=torch.int64)
- slot_mapping = torch.zeros(max_bs, dtype=torch.int32)
- context_lens = torch.zeros(max_bs, dtype=torch.int32)
- block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32)
- outputs = torch.zeros(max_bs, hf_config.hidden_size)
+ input_ids = torch.zeros(max_bs, dtype=torch.int64, device=model_device)
+ positions = torch.zeros(max_bs, dtype=torch.int64, device=model_device)
+ slot_mapping = torch.zeros(max_bs, dtype=torch.int32, device=model_device)
+ context_lens = torch.zeros(max_bs, dtype=torch.int32, device=model_device)
+ block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32, device=model_device)
+ outputs = torch.zeros(max_bs, hf_config.hidden_size, device=model_device)
self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16))
self.graphs = {}
self.graph_pool = None
diff --git a/shared/llm_engines/nanovllm/layers/linear.py b/shared/llm_engines/nanovllm/layers/linear.py
index 3476cbb..ab19766 100644
--- a/shared/llm_engines/nanovllm/layers/linear.py
+++ b/shared/llm_engines/nanovllm/layers/linear.py
@@ -117,9 +117,20 @@ class LinearBase(nn.Module):
def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):
device = self.weight.device
- shard = self._shard_weight(loaded_weight, loaded_shard_id).contiguous()
- q = shard.to(device=device, dtype=torch.int8, non_blocking=True)
- self.qweight_data = q
+ shard = self._shard_weight(loaded_weight, loaded_shard_id)
+ if self.qweight_t.numel() == 0:
+ self.qweight_t = torch.empty(
+ (self.input_size, self.output_size),
+ dtype=torch.int8,
+ device=device,
+ )
+ if shard.dtype != torch.int8:
+ shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)
+ else:
+ shard = shard.to(device=device, non_blocking=True)
+ # Store directly in transposed layout to avoid an extra full-size transpose pass
+ # after loading.
+ self.qweight_t.t().copy_(shard)
shard_key = loaded_shard_id if loaded_shard_id is not None else 0
self._quant_data_loaded.add(shard_key)
@@ -141,15 +152,20 @@ class LinearBase(nn.Module):
self.output_scale = loaded_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)
def finalize_quantized(self):
- if self.qweight_data.numel() == 0 or self.qweight_scale.numel() == 0:
+ if self.qweight_scale.numel() == 0:
return
if len(self._quant_data_loaded) < self._quant_expected_shards:
return
if len(self._quant_scale_loaded) < self._quant_expected_shards:
return
- # Keep int8 weights in KxN layout for int8 GEMM paths.
- self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()
- self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)
+ if self.qweight_data.numel() == 0 and self.qweight_t.numel() == 0:
+ return
+ if self.qweight_t.numel() == 0:
+ # Fallback path for loaders that still filled qweight_data (older checkpoints
+ # or custom loaders). Keep memory usage low by releasing qweight_data immediately
+ # after conversion.
+ self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()
+ self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)
self.qweight_scale_fp32 = self.qweight_scale.to(dtype=torch.float32)
if not torch.is_tensor(self.input_scale):
raise RuntimeError(f"Invalid input_scale type: {type(self.input_scale)}")
@@ -186,6 +202,7 @@ class LinearBase(nn.Module):
raise RuntimeError("quanto.qbytes_mm op unavailable for int8 fallback path")
if self.qweight_t.numel() == 0 and self.qweight_data.numel() != 0:
self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()
+ self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)
qweight = self.qweight_t.transpose(0, 1).contiguous()
scales = self.qweight_scale_fp32
if scales.numel() == 0:
@@ -206,6 +223,8 @@ class LinearBase(nn.Module):
device = self.qweight_scale.device
else:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ if getattr(device, "type", None) == "meta":
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if self.weight.numel() != 0:
self.weight = nn.Parameter(torch.empty(0, device=device, dtype=torch.bfloat16), requires_grad=False)
@@ -301,12 +320,15 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):
device = self.weight.device
- if self.qweight_data.numel() == 0:
- self.qweight_data = torch.empty((self.output_size, self.input_size), dtype=torch.int8, device=device)
+ if self.qweight_t.numel() == 0:
+ self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)
shard_offset, shard_size = self._merged_shard_meta(int(loaded_shard_id))
- param_slice = self.qweight_data.narrow(0, shard_offset, shard_size)
shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]
- param_slice.copy_(shard.to(device=device, dtype=torch.int8, non_blocking=True))
+ if shard.dtype != torch.int8:
+ shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)
+ else:
+ shard = shard.to(device=device, non_blocking=True)
+ self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())
self._quant_data_loaded.add(int(loaded_shard_id))
def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):
@@ -363,12 +385,15 @@ class QKVParallelLinear(ColumnParallelLinear):
def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):
device = self.weight.device
- if self.qweight_data.numel() == 0:
- self.qweight_data = torch.empty((self.output_size, self.input_size), dtype=torch.int8, device=device)
+ if self.qweight_t.numel() == 0:
+ self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)
shard_offset, shard_size = self._qkv_offset_size(str(loaded_shard_id))
- param_slice = self.qweight_data.narrow(0, shard_offset, shard_size)
shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]
- param_slice.copy_(shard.to(device=device, dtype=torch.int8, non_blocking=True))
+ if shard.dtype != torch.int8:
+ shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)
+ else:
+ shard = shard.to(device=device, non_blocking=True)
+ self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())
self._quant_data_loaded.add(str(loaded_shard_id))
def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):
diff --git a/shared/llm_engines/nanovllm/layers/rotary_embedding.py b/shared/llm_engines/nanovllm/layers/rotary_embedding.py
index 998d116..acaeffd 100644
--- a/shared/llm_engines/nanovllm/layers/rotary_embedding.py
+++ b/shared/llm_engines/nanovllm/layers/rotary_embedding.py
@@ -25,14 +25,30 @@ class RotaryEmbedding(nn.Module):
) -> None:
super().__init__()
self.head_size = head_size
+ self.rotary_dim = rotary_dim
+ self.max_position_embeddings = max_position_embeddings
+ self.base = float(base)
assert rotary_dim == head_size
- inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
- t = torch.arange(max_position_embeddings, dtype=torch.float)
+ cache = self._build_cache(device=None)
+ self.register_buffer("cos_sin_cache", cache, persistent=False)
+
+ def _build_cache(self, device: torch.device | str | None) -> torch.Tensor:
+ inv_freq = 1.0 / (
+ self.base ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32, device=device) / self.rotary_dim)
+ )
+ t = torch.arange(self.max_position_embeddings, dtype=torch.float32, device=device)
freqs = torch.einsum("i,j -> ij", t, inv_freq)
cos = freqs.cos()
sin = freqs.sin()
- cache = torch.cat((cos, sin), dim=-1).unsqueeze_(1)
- self.register_buffer("cos_sin_cache", cache, persistent=False)
+ return torch.cat((cos, sin), dim=-1).unsqueeze_(1)
+
+ def materialize_cache(self, device: torch.device | str | None = None):
+ target_device = device
+ if target_device is None:
+ target_device = self.cos_sin_cache.device
+ if getattr(target_device, "type", None) == "meta":
+ target_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.cos_sin_cache = self._build_cache(target_device)
@torch.compile
def forward(
diff --git a/shared/llm_engines/nanovllm/layers/sampler.py b/shared/llm_engines/nanovllm/layers/sampler.py
index 41fcb27..8753882 100644
--- a/shared/llm_engines/nanovllm/layers/sampler.py
+++ b/shared/llm_engines/nanovllm/layers/sampler.py
@@ -97,6 +97,7 @@ class Sampler(nn.Module):
top_ps: Optional[torch.Tensor] = None,
repetition_penalties: Optional[torch.Tensor] = None,
input_ids: Optional[torch.Tensor] = None,
+ generator: Optional[torch.Generator] = None,
):
"""
Sample tokens from logits with optional top-k and top-p filtering.
@@ -118,5 +119,6 @@ class Sampler(nn.Module):
if invalid_rows.any():
logits[invalid_rows, 0] = 0.0
probs = torch.softmax(logits, dim=-1)
- sample_tokens = probs.div_(torch.empty_like(probs).exponential_(1).clamp_min_(1e-10)).argmax(dim=-1)
+ noise = torch.empty_like(probs).exponential_(1, generator=generator).clamp_min_(1e-10)
+ sample_tokens = probs.div_(noise).argmax(dim=-1)
return sample_tokens
diff --git a/shared/llm_engines/nanovllm/models/__init__.py b/shared/llm_engines/nanovllm/models/__init__.py
new file mode 100644
index 0000000..692e251
--- /dev/null
+++ b/shared/llm_engines/nanovllm/models/__init__.py
@@ -0,0 +1,52 @@
+import importlib
+from typing import Iterable
+from torch import nn
+
+
+_ARCH_REGISTRY = {
+ "Qwen3ForCausalLM": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),
+}
+
+_MODEL_TYPE_REGISTRY = {
+ "qwen3": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),
+}
+
+
+def _load_model_cls(module_name: str, class_name: str):
+ module = importlib.import_module(module_name)
+ model_cls = getattr(module, class_name, None)
+ if model_cls is None:
+ raise RuntimeError(f"Model class '{class_name}' was not found in '{module_name}'.")
+ if not isinstance(model_cls, type) or not issubclass(model_cls, nn.Module):
+ raise RuntimeError(f"Resolved '{module_name}.{class_name}' is not a torch.nn.Module subclass.")
+ return model_cls
+
+
+def _iter_architectures(hf_config) -> Iterable[str]:
+ architectures = getattr(hf_config, "architectures", None)
+ if isinstance(architectures, (list, tuple)):
+ for architecture in architectures:
+ if isinstance(architecture, str) and len(architecture) > 0:
+ yield architecture
+
+
+def resolve_model_class(hf_config):
+ for architecture in _iter_architectures(hf_config):
+ entry = _ARCH_REGISTRY.get(architecture, None)
+ if entry is not None:
+ return _load_model_cls(entry[0], entry[1])
+
+ model_type = str(getattr(hf_config, "model_type", "")).strip().lower()
+ if model_type:
+ entry = _MODEL_TYPE_REGISTRY.get(model_type, None)
+ if entry is not None:
+ return _load_model_cls(entry[0], entry[1])
+
+ supported_architectures = ", ".join(sorted(_ARCH_REGISTRY.keys()))
+ supported_model_types = ", ".join(sorted(_MODEL_TYPE_REGISTRY.keys()))
+ found_architectures = ", ".join(list(_iter_architectures(hf_config))) or "<none>"
+ raise RuntimeError(
+ "nanovllm does not support this model architecture. "
+ f"Found architectures: {found_architectures}; model_type: '{model_type or '<none>'}'. "
+ f"Supported architectures: {supported_architectures}; supported model_type values: {supported_model_types}."
+ )
diff --git a/shared/llm_engines/nanovllm/models/qwen3.py b/shared/llm_engines/nanovllm/models/qwen3.py
index 3f1c845..7b98783 100644
--- a/shared/llm_engines/nanovllm/models/qwen3.py
+++ b/shared/llm_engines/nanovllm/models/qwen3.py
@@ -204,8 +204,6 @@ class Qwen3ForCausalLM(nn.Module):
super().__init__()
self.model = Qwen3Model(config)
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
- if config.tie_word_embeddings:
- self.lm_head.weight.data = self.model.embed_tokens.weight.data
# Proxy attributes for weight loading compatibility
# Some model weights use "embed_tokens" instead of "model.embed_tokens"
diff --git a/shared/llm_engines/nanovllm/utils/loader.py b/shared/llm_engines/nanovllm/utils/loader.py
index 17ac800..92bf8b1 100644
--- a/shared/llm_engines/nanovllm/utils/loader.py
+++ b/shared/llm_engines/nanovllm/utils/loader.py
@@ -1,4 +1,6 @@
+import gc
import os
+from collections import OrderedDict
from glob import glob
import torch
from torch import nn
@@ -6,52 +8,99 @@ from safetensors import safe_open
def default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor):
+ if param.is_meta:
+ raise RuntimeError("default_weight_loader cannot copy into meta parameter directly.")
+ if loaded_weight.device != param.device or loaded_weight.dtype != param.dtype:
+ loaded_weight = loaded_weight.to(
+ device=param.device,
+ dtype=param.dtype,
+ non_blocking=True,
+ )
param.data.copy_(loaded_weight)
-def _get_parameter_safe(model: nn.Module, weight_name: str):
- """
- Try to get parameter from model, handling name mismatches.
+def _coerce_loaded_weight_to_param(param: nn.Parameter, loaded_weight: torch.Tensor) -> torch.Tensor:
+ target_device = param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ if loaded_weight.device != target_device:
+ loaded_weight = loaded_weight.to(device=target_device, non_blocking=True)
+ if loaded_weight.dtype != param.dtype:
+ loaded_weight = loaded_weight.to(dtype=param.dtype, non_blocking=True)
+ return loaded_weight
- Some models have nested structure (e.g., Qwen3ForCausalLM has model.embed_tokens)
- but weight files may have flat names (embed_tokens.weight).
- """
- # Try direct access first
- try:
- return model.get_parameter(weight_name)
- except AttributeError:
- pass
- # Try with 'model.' prefix (for nested model structure)
- try:
- prefixed_name = f"model.{weight_name}"
- return model.get_parameter(prefixed_name)
- except AttributeError:
- pass
+def _restore_param_attributes(new_param: nn.Parameter, src_param: nn.Parameter) -> None:
+ for attr_name, attr_value in src_param.__dict__.items():
+ setattr(new_param, attr_name, attr_value)
- # Try removing 'model.' prefix
+
+def _replace_meta_parameter(
+ parent: nn.Module,
+ param_name: str,
+ loaded_weight: torch.Tensor,
+ src_param: nn.Parameter,
+) -> nn.Parameter:
+ new_param = nn.Parameter(
+ loaded_weight,
+ requires_grad=src_param.requires_grad,
+ )
+ parent._parameters[param_name] = new_param
+ _restore_param_attributes(new_param, src_param)
+ return new_param
+
+
+def _replace_tied_meta_parameter(
+ model: nn.Module,
+ old_param: nn.Parameter,
+ loaded_weight: torch.Tensor,
+) -> nn.Parameter:
+ new_param = nn.Parameter(
+ loaded_weight,
+ requires_grad=old_param.requires_grad,
+ )
+ _restore_param_attributes(new_param, old_param)
+ for module in model.modules():
+ for name, param in list(module._parameters.items()):
+ if param is old_param:
+ module._parameters[name] = new_param
+ return new_param
+
+
+def _should_replace_param_globally(model: nn.Module, param: nn.Parameter) -> bool:
+ tied_ids = getattr(model, "_vllm_tied_param_ids", None)
+ return bool(tied_ids) and id(param) in tied_ids
+
+
+def _candidate_weight_names(weight_name: str):
+ candidates = [weight_name]
if weight_name.startswith("model."):
- try:
- unprefixed_name = weight_name[6:] # Remove 'model.' prefix
- return model.get_parameter(unprefixed_name)
- except AttributeError:
- pass
+ candidates.append(weight_name[6:])
+ else:
+ candidates.append(f"model.{weight_name}")
+ return candidates
+
+def _get_parameter_target(model: nn.Module, weight_name: str):
+ for candidate in _candidate_weight_names(weight_name):
+ if "." in candidate:
+ parent_name, param_name = candidate.rsplit(".", 1)
+ else:
+ parent_name, param_name = "", candidate
+ try:
+ parent = model if parent_name == "" else model.get_submodule(parent_name)
+ except Exception:
+ continue
+ param = parent._parameters.get(param_name)
+ if param is not None and isinstance(param, nn.Parameter):
+ return parent, param_name, param
return None
def _get_submodule_safe(model: nn.Module, module_name: str):
- try:
- return model.get_submodule(module_name)
- except Exception:
- pass
- try:
- return model.get_submodule(f"model.{module_name}")
- except Exception:
- pass
- if module_name.startswith("model."):
+ for candidate in _candidate_weight_names(module_name):
+ if candidate == "":
+ return model
try:
- return model.get_submodule(module_name[6:])
+ return model.get_submodule(candidate)
except Exception:
pass
return None
@@ -70,60 +119,86 @@ class WeightStore:
self.files = _list_safetensor_files(path)
if not self.files:
raise FileNotFoundError(f"No .safetensors files found in {path}")
- self._file_handles = {}
self._weight_to_file = {}
+ self._file_to_weight_names = OrderedDict()
self._pinned_weights = {}
self.is_quanto_int8 = False
for file in self.files:
- with safe_open(file, "pt", "cpu") as f:
+ with safe_open(file, "pt", "cpu", writable_tensors=False) as f:
for key in f.keys():
self._weight_to_file[key] = file
+ self._file_to_weight_names.setdefault(file, []).append(key)
if key.endswith(".weight._data"):
self.is_quanto_int8 = True
if self.mode == "pinned":
self._preload_pinned()
- def _get_handle(self, file_path: str):
- handle = self._file_handles.get(file_path)
- if handle is None:
- handle = safe_open(file_path, "pt", "cpu")
- self._file_handles[file_path] = handle
- return handle
-
def _preload_pinned(self):
- for key, file in self._weight_to_file.items():
- handle = self._get_handle(file)
- tensor = handle.get_tensor(key)
- if tensor.device.type != "cpu":
- tensor = tensor.cpu()
- if not tensor.is_pinned():
- tensor = tensor.pin_memory()
- self._pinned_weights[key] = tensor
+ for file, keys in self._file_to_weight_names.items():
+ with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:
+ for key in keys:
+ tensor = handle.get_tensor(key)
+ if tensor.device.type != "cpu":
+ tensor = tensor.cpu()
+ if not tensor.is_pinned():
+ tensor = tensor.pin_memory()
+ self._pinned_weights[key] = tensor
+
+ def close(self):
+ self._weight_to_file.clear()
+ self._file_to_weight_names.clear()
+ self._pinned_weights.clear()
+
+ def __del__(self):
+ try:
+ self.close()
+ except Exception:
+ pass
def get_tensor(self, key: str) -> torch.Tensor:
if self.mode == "pinned":
return self._pinned_weights[key]
- handle = self._get_handle(self._weight_to_file[key])
- return handle.get_tensor(key)
+ file = self._weight_to_file.get(key)
+ if file is None:
+ raise KeyError(key)
+ with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:
+ return handle.get_tensor(key)
-def load_model(model: nn.Module, path: str, weight_store: WeightStore | None = None):
+def load_model(
+ model: nn.Module,
+ path: str,
+ weight_store: WeightStore | None = None,
+ clone_loaded_tensors: bool = False,
+):
packed_modules_mapping = getattr(model, "packed_modules_mapping", {})
if weight_store is None:
safetensor_files = _list_safetensor_files(path)
if not safetensor_files:
raise FileNotFoundError(f"No .safetensors files found in {path}")
for file in safetensor_files:
- with safe_open(file, "pt", "cpu") as f:
+ with safe_open(file, "pt", "cpu", writable_tensors=False) as f:
for weight_name in f.keys():
tensor = f.get_tensor(weight_name)
_apply_weight(model, packed_modules_mapping, weight_name, tensor)
_finalize_quantized_modules(model)
return
-
- for weight_name in weight_store._weight_to_file.keys():
- tensor = weight_store.get_tensor(weight_name)
- _apply_weight(model, packed_modules_mapping, weight_name, tensor)
+ for file in weight_store.files:
+ file_weights = weight_store._file_to_weight_names.get(file, [])
+ if not file_weights:
+ continue
+ with safe_open(file, "pt", "cpu", writable_tensors=False) as f:
+ for weight_name in file_weights:
+ tensor = f.get_tensor(weight_name)
+ if clone_loaded_tensors:
+ tensor = tensor.clone()
+ try:
+ _apply_weight(model, packed_modules_mapping, weight_name, tensor)
+ finally:
+ del tensor
+ if torch.cuda.is_available():
+ torch.cuda.synchronize()
+ gc.collect()
_finalize_quantized_modules(model)
@@ -138,79 +213,78 @@ def _apply_weight(model: nn.Module, packed_modules_mapping, weight_name: str, te
elif weight_name.endswith(".output_scale"):
quant_suffix = "output_scale"
- for k in packed_modules_mapping:
- if k in weight_name:
- v, shard_id = packed_modules_mapping[k]
- mapped_name = weight_name.replace(k, v)
- if quant_suffix is not None:
- module_name = mapped_name
- if quant_suffix == "qdata":
- module_name = mapped_name[: -len(".weight._data")]
- elif quant_suffix == "qscale":
- module_name = mapped_name[: -len(".weight._scale")]
- elif quant_suffix == "input_scale":
- module_name = mapped_name[: -len(".input_scale")]
- elif quant_suffix == "output_scale":
- module_name = mapped_name[: -len(".output_scale")]
- module = _get_submodule_safe(model, module_name)
- if module is None:
- print(f"[loader] Warning: Module not found: {module_name}")
- return
- if quant_suffix == "qdata":
- loader_fn = getattr(module, "quant_weight_data_loader", None)
- elif quant_suffix == "qscale":
- loader_fn = getattr(module, "quant_weight_scale_loader", None)
- elif quant_suffix == "input_scale":
- loader_fn = getattr(module, "quant_input_scale_loader", None)
- else:
- loader_fn = getattr(module, "quant_output_scale_loader", None)
- if loader_fn is None:
- print(f"[loader] Warning: Quant loader not found on module: {module_name}")
- return
- if quant_suffix in ("qdata", "qscale"):
- loader_fn(tensor, shard_id)
- else:
- loader_fn(tensor)
- return
- param_name = mapped_name
- param = _get_parameter_safe(model, param_name)
- if param is None:
- print(f"[loader] Warning: Parameter not found: {param_name}")
- return
- weight_loader = getattr(param, "weight_loader")
- weight_loader(param, tensor, shard_id)
- return
+ mapped_name = weight_name
+ shard_id = None
+ for src_name, mapped in packed_modules_mapping.items():
+ if src_name in weight_name:
+ mapped_target = mapped[0] if isinstance(mapped, tuple) else mapped
+ mapped_name = weight_name.replace(src_name, mapped_target)
+ shard_id = mapped[1] if isinstance(mapped, tuple) and len(mapped) > 1 else None
+ break
+
if quant_suffix is not None:
if quant_suffix == "qdata":
- module_name = weight_name[: -len(".weight._data")]
+ module_name = mapped_name[:-len(".weight._data")]
+ loader_name = "quant_weight_data_loader"
elif quant_suffix == "qscale":
- module_name = weight_name[: -len(".weight._scale")]
+ module_name = mapped_name[:-len(".weight._scale")]
+ loader_name = "quant_weight_scale_loader"
elif quant_suffix == "input_scale":
- module_name = weight_name[: -len(".input_scale")]
+ module_name = mapped_name[:-len(".input_scale")]
+ loader_name = "quant_input_scale_loader"
else:
- module_name = weight_name[: -len(".output_scale")]
+ module_name = mapped_name[:-len(".output_scale")]
+ loader_name = "quant_output_scale_loader"
module = _get_submodule_safe(model, module_name)
if module is None:
print(f"[loader] Warning: Module not found: {module_name}")
return
- if quant_suffix == "qdata":
- loader_fn = getattr(module, "quant_weight_data_loader", None)
- elif quant_suffix == "qscale":
- loader_fn = getattr(module, "quant_weight_scale_loader", None)
- elif quant_suffix == "input_scale":
- loader_fn = getattr(module, "quant_input_scale_loader", None)
- else:
- loader_fn = getattr(module, "quant_output_scale_loader", None)
+ loader_fn = getattr(module, loader_name, None)
if loader_fn is None:
print(f"[loader] Warning: Quant loader not found on module: {module_name}")
return
- loader_fn(tensor)
+ if shard_id is not None and quant_suffix in ("qdata", "qscale"):
+ loader_fn(tensor, shard_id)
+ else:
+ loader_fn(tensor)
+ return
+
+ target = _get_parameter_target(model, mapped_name)
+ if target is None:
+ print(f"[loader] Warning: Parameter not found: {mapped_name}")
return
- param = _get_parameter_safe(model, weight_name)
- if param is None:
- print(f"[loader] Warning: Parameter not found: {weight_name}")
+ parent, param_name, param = target
+ weight_loader = getattr(param, "weight_loader", None)
+
+ if shard_id is not None and weight_loader is not None:
+ if param.is_meta:
+ replacement_device = (
+ param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ )
+ replacement = torch.empty(param.shape, device=replacement_device, dtype=param.dtype)
+ if _should_replace_param_globally(model, param):
+ param = _replace_tied_meta_parameter(model, param, replacement)
+ else:
+ param = _replace_meta_parameter(parent, param_name, replacement, param)
+ try:
+ weight_loader(param, tensor, shard_id)
+ except TypeError:
+ weight_loader(param, tensor)
+ return
+
+ if param.is_meta:
+ loaded_weight = _coerce_loaded_weight_to_param(param, tensor)
+ if loaded_weight.shape != param.shape:
+ raise RuntimeError(
+ f"Weight shape mismatch for '{mapped_name}': expected {tuple(param.shape)}, got {tuple(loaded_weight.shape)}"
+ )
+ if _should_replace_param_globally(model, param):
+ _replace_tied_meta_parameter(model, param, loaded_weight)
+ else:
+ _replace_meta_parameter(parent, param_name, loaded_weight, param)
return
- weight_loader = getattr(param, "weight_loader", default_weight_loader)
+ if weight_loader is None:
+ weight_loader = default_weight_loader
weight_loader(param, tensor)
diff --git a/shared/utils/transformers_fast_tokenizer_patch.py b/shared/utils/transformers_fast_tokenizer_patch.py
index c1f0b8a..21c020d 100644
--- a/shared/utils/transformers_fast_tokenizer_patch.py
+++ b/shared/utils/transformers_fast_tokenizer_patch.py
@@ -6,6 +6,7 @@ import sys
_PATCH_ALLOWED_PATHS = None
_ORIG_FAST_INIT = None
+_DISABLE_FULL_TOKENIZER_PICKLE_CACHE = True # Temporary: disable Python pickle tokenizer cache load/dump.
def _normalize_path(path):
@@ -273,6 +274,8 @@ def _meta_matches(meta, tokenizer_dir):
def _load_full_tokenizer_cache(tokenizer_dir, cache_tag=None):
+ if _DISABLE_FULL_TOKENIZER_PICKLE_CACHE:
+ return None
cache_file, meta_file = _cache_paths(tokenizer_dir, cache_tag=cache_tag)
if not os.path.isfile(cache_file) or not os.path.isfile(meta_file):
return None
@@ -287,6 +290,8 @@ def _load_full_tokenizer_cache(tokenizer_dir, cache_tag=None):
def _save_full_tokenizer_cache(tokenizer_dir, tokenizer, cache_tag=None):
+ if _DISABLE_FULL_TOKENIZER_PICKLE_CACHE:
+ return
cache_file, meta_file = _cache_paths(tokenizer_dir, cache_tag=cache_tag)
meta = {
"py_version": list(sys.version_info[:3]),
diff --git a/wgp.py b/wgp.py
index 376c89d..e44c2e6 100644
--- a/wgp.py
+++ b/wgp.py
@@ -94,7 +94,7 @@ AUTOSAVE_TEMPLATE_PATH = AUTOSAVE_FILENAME
CONFIG_FILENAME = "wgp_config.json"
PROMPT_VARS_MAX = 10
target_mmgp_version = "3.7.4"
-WanGP_version = "10.83"
+WanGP_version = "10.84"
settings_version = 2.51
max_source_video_frames = 3000
prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None
@@ -170,13 +170,6 @@ def clear_gen_cache():
def release_model():
global wan_model, offloadobj, reload_needed
- if wan_model is not None:
- close_fn = getattr(wan_model, "close", None)
- if callable(close_fn):
- try:
- close_fn()
- except Exception:
- pass
wan_model = None
clear_gen_cache()
if "_cache" in offload.shared_state:
@@ -3544,6 +3537,7 @@ def setup_prompt_enhancer(pipe, kwargs):
fl.locate_file("Llama3_2/Llama3_2_quanto_bf16_int8.safetensors"),
defaultConfigPath=fl.locate_file("Llama3_2/config.json", error_if_none=False),
configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},
+ writable_tensors=False,
)
prompt_enhancer_llm_model._validate_model_kwargs = lambda *_args, **_kwargs: None
prompt_enhancer_llm_model._offload_hooks = ["generate"]
@@ -3559,6 +3553,7 @@ def setup_prompt_enhancer(pipe, kwargs):
forcedConfigPath=fl.locate_file("llama-joycaption-beta-one-hf-llava/llama_config.json", error_if_none=False),
configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},
preprocess_sd=preprocess_sd,
+ writable_tensors=False,
)
prompt_enhancer_llm_tokenizer = AutoTokenizer.from_pretrained(fl.locate_folder("llama-joycaption-beta-one-hf-llava"))
@@ -5779,7 +5774,7 @@ def generate_video(
torch.set_grad_enabled(False)
if mode.startswith("edit_"):
edit_video(send_cmd, state, mode, video_source, seed, temporal_upsampling, spatial_upsampling, film_grain_intensity, film_grain_saturation, MMAudio_setting, MMAudio_prompt, MMAudio_neg_prompt, repeat_generation, audio_source)
- return
+ return True
with lock:
file_list = gen["file_list"]
file_settings_list = gen["file_settings_list"]
@@ -5833,7 +5828,6 @@ def generate_video(
profile = compute_profile(override_profile, output_type)
enhancer_mode = server_config.get("enhancer_mode", 1)
if model_type != transformer_type or reload_needed or profile != loaded_profile:
- wan_model = None
release_model()
send_cmd("status", f"Loading model {get_model_name(model_type)}...")
wan_model, offloadobj = load_models(
@@ -5846,7 +5840,7 @@ def generate_video(
reload_needed= False
if args.test:
send_cmd("info", "Test mode: model loaded, skipping generation.")
- return
+ return True
overridden_attention = override_attention if len(override_attention) else get_overridden_attention(model_type)
# if overridden_attention is not None and overridden_attention != attention_mode: print(f"Attention mode has been overriden to {overridden_attention} for model type '{model_type}'")
attn = overridden_attention if overridden_attention is not None else attention_mode
@@ -5855,7 +5849,7 @@ def generate_video(
elif not attn in attention_modes_supported:
send_cmd("info", f"You have selected attention mode '{attention_mode}'. However it is not installed or supported on your system. You should either install it or switch to the default 'sdpa' attention.")
send_cmd("exit")
- return
+ return True
width, height = resolution.split("x")
width, height = int(width) // block_size * block_size, int(height) // block_size * block_size
@@ -6135,7 +6129,7 @@ def generate_video(
wan_model._interrupt = False
abort = False
if gen.get("abort", False):
- return
+ return True
# gen["abort"] = False
gen["prompt"] = prompt
repeat_no = 0
@@ -6663,7 +6657,7 @@ def generate_video(
print('\n'.join(tb))
send_cmd("error", new_error)
clear_status(state)
- return
+ return False
src_video = src_video2 = src_mask = src_mask2 = None
if skip_steps_cache != None :
skip_steps_cache.previous_residual = None
@@ -6950,6 +6944,7 @@ def generate_video(
cleanup_temp_audio_files(control_audio_tracks + source_audio_tracks)
remove_temp_filenames(temp_filenames_list)
+ return True
def prepare_generate_video(state):
@@ -7129,7 +7124,7 @@ def process_tasks(state):
filtered_params = {k: v for k, v in params.items() if k in expected_args}
plugin_data = task.pop('plugin_data', {})
- generate_video(task, send_cmd, plugin_data=plugin_data, **filtered_params)
+ success = generate_video(task, send_cmd, plugin_data=plugin_data, **filtered_params)
except Exception as e:
tb = traceback.format_exc().split('\n')[:-1]
@@ -7145,7 +7140,7 @@ def process_tasks(state):
gen["early_stop"] = False
gen["early_stop_forwarded"] = False
-
+ if not success: break
with lock:
queue[:] = [item for item in queue if item['id'] != task_id]
update_global_queue_ref(queue)
@@ -8819,7 +8814,6 @@ def preload_model_when_switching(state):
if "S" in preload_model_policy:
model_type = get_state_model_type(state)
if model_type != transformer_type:
- wan_model = None
release_model()
model_filename = get_model_name(model_type)
yield f"Loading model {model_filename}..."
@@ -8836,7 +8830,6 @@ def unload_model_if_needed(state):
global wan_model
if "U" in preload_model_policy:
if wan_model != None:
- wan_model = None
release_model()
def all_letters(source_str, letters):