Skip to content
Python

refactory vllm to use mmgp, ace step 1.5 cleanup

DeepBeepMeep10ef884e4e51
diff --git a/models/TTS/ace_step15/pipeline_ace_step15.py b/models/TTS/ace_step15/pipeline_ace_step15.py
index 4bc9e2b..4e6a6b3 100644
--- a/models/TTS/ace_step15/pipeline_ace_step15.py
+++ b/models/TTS/ace_step15/pipeline_ace_step15.py
@@ -7,20 +7,21 @@ import math
import os
import random
import re
-import sys
from typing import Any
import torch
import torchaudio
import yaml
from tqdm import tqdm
-from transformers import AutoTokenizer, Qwen3ForCausalLM, Qwen3Model
+from transformers import AutoTokenizer, Qwen3ForCausalLM as HfQwen3ForCausalLM, Qwen3Model
from mmgp import offload
from shared.utils.text_encoder_cache import TextEncoderCache
from .models.autoencoder_oobleck import AutoencoderOobleck
from .models.ace_step15_hf import AceStepConditionGenerationModel
+from .qwen3_audio_codes import generate_audio_codes_with_engine_sampling
+from .qwen3_lm_engines import Qwen3LegacyEngine, Qwen3LmEngine, Qwen3VllmEngine
_DEFAULT_TIMBRE = [
-1.3672e-01, -1.5820e-01, 5.8594e-01, -5.7422e-01, 3.0273e-02,
@@ -111,8 +112,6 @@ class ACEStep15Pipeline:
enable_lm: bool = True,
ignore_lm_cache_seed: bool = False,
lm_decoder_engine: str = "legacy",
- lm_vllm_weight_mode: str = "lazy",
- lm_vllm_free_mmap_after_cuda_copy: bool = True,
device=None,
dtype=torch.bfloat16,
):
@@ -137,26 +136,15 @@ class ACEStep15Pipeline:
self.lm_tokenizer_dir = lm_tokenizer_dir
self.silence_latent_path = silence_latent_path
self.ignore_lm_cache_seed = bool(ignore_lm_cache_seed)
- self.lm_engine = (lm_decoder_engine or "legacy").strip().lower()
- if self.lm_engine not in ("legacy", "pt", "vllm"):
- self.lm_engine = "legacy"
- self.lm_vllm_weight_mode = (lm_vllm_weight_mode or "lazy").strip().lower()
- if self.lm_engine == "vllm" and self.lm_vllm_weight_mode not in ("lazy", "pinned"):
- raise ValueError(
- f"Unsupported Ace Step vllm LM weight mode '{self.lm_vllm_weight_mode}'. Supported: lazy, pinned."
- )
- self.lm_vllm_free_mmap_after_cuda_copy = bool(lm_vllm_free_mmap_after_cuda_copy)
+ self.lm_engine = str(lm_decoder_engine or "legacy").strip().lower()
+ if self.lm_engine not in {"legacy", "vllm"}:
+ raise ValueError(f"Unsupported LM engine '{lm_decoder_engine}'. Expected one of: legacy, vllm.")
self._interrupt = False
self._early_stop = False
self.loaded = False
self._latent_hop_length = 1920
- self.lm_code_cache = TextEncoderCache()
- self._lm_engine_impl = None
- self._lm_create_engine_fn = None
- self._lm_legacy_generate_fn = None
- self._lm_legacy_text_fn = None
self._ref_metadata_processor = None
self._ref_metadata_processor_class = None
self._ref_caption_postprocess_fn = None
@@ -165,12 +153,13 @@ class ACEStep15Pipeline:
self._init_lm_hint_modules()
self._load_tokenizers()
self._load_text_encoder_2()
- if self.enable_lm and self.lm_engine != "vllm":
- self._load_lm()
- else:
- self.lm_model = None
self._load_silence_latent()
+ self.lm_code_cache = TextEncoderCache()
+ self._lm_engine_impl: Qwen3LmEngine | None = None
+ self._lm_last_failure_reason = ""
+ self.lm_model = None
if self.enable_lm:
+ self._load_lm()
self._init_lm_engine()
self.loaded = True
@@ -312,66 +301,45 @@ class ACEStep15Pipeline:
def _load_lm(self):
config_path = os.path.join(os.path.dirname(self.lm_weights_path), "config.json")
+
def _remap_lm_state_dict(state_dict, quantization_map=None, tied_weights_map=None):
- # AceStep 5Hz LM weights are stored without a `model.` prefix.
- if any(key.startswith("model.") for key in state_dict.keys()):
- if "lm_head.weight" not in state_dict and "model.embed_tokens.weight" in state_dict:
- state_dict["lm_head.weight"] = state_dict["model.embed_tokens.weight"]
- return state_dict, quantization_map, tied_weights_map
- remapped = {f"model.{key}": value for key, value in state_dict.items()}
- if "model.embed_tokens.weight" in remapped and "lm_head.weight" not in remapped:
- remapped["lm_head.weight"] = remapped["model.embed_tokens.weight"]
- return remapped, quantization_map, tied_weights_map
+ embed_key = "model.embed_tokens.weight"
+ if "lm_head.weight" not in state_dict and embed_key in state_dict:
+ state_dict["lm_head.weight"] = state_dict[embed_key]
+ return state_dict, quantization_map, tied_weights_map
+
+ if self.lm_engine == "vllm":
+ from shared.llm_engines.nanovllm.models.qwen3 import Qwen3ForCausalLM as NanoQwen3ForCausalLM
+ model_class = NanoQwen3ForCausalLM
+ unlimited_budget = True
+ else:
+ model_class = HfQwen3ForCausalLM
+ unlimited_budget = False
self.lm_model = offload.fast_load_transformers_model(
self.lm_weights_path,
- modelClass=Qwen3ForCausalLM,
+ modelClass=model_class,
defaultConfigPath=config_path,
default_dtype=self.dtype,
preprocess_sd=_remap_lm_state_dict,
writable_tensors=False,
ignore_unused_weights=True,
)
+ if unlimited_budget: self.lm_model._budget = 0
self.lm_model.eval()
- self._disable_lm_compile_for_mmgp()
-
- def _disable_lm_compile_for_mmgp(self):
- if self.lm_model is None:
- return
- try:
- self.lm_model._compile_me = False
- for submodule in self.lm_model.modules():
- submodule._compile_me = False
- except Exception:
- return
-
- def _ensure_lm_module_loaded(self):
- if self._lm_create_engine_fn is not None and self._lm_legacy_generate_fn is not None and self._lm_legacy_text_fn is not None:
- return
- from .qwen3_audio_codes import create_qwen3_lm_engine, generate_audio_codes_legacy, generate_text_legacy
-
- self._lm_create_engine_fn = create_qwen3_lm_engine
- self._lm_legacy_generate_fn = generate_audio_codes_legacy
- self._lm_legacy_text_fn = generate_text_legacy
def _init_lm_engine(self):
- self._ensure_lm_module_loaded()
- self._lm_engine_impl = self._lm_create_engine_fn(
- engine_name=self.lm_engine,
- model=self.lm_model,
- tokenizer=self.lm_tokenizer,
- device=self.device,
- lm_weights_path=self.lm_weights_path,
- audio_code_mask=getattr(self, "_audio_code_mask", None),
- audio_code_token_map=getattr(self, "_audio_code_token_map", {}),
- free_mmap_after_cuda_copy=self.lm_vllm_free_mmap_after_cuda_copy,
- weight_load_mode=self.lm_vllm_weight_mode,
- )
- if self.lm_engine in ("pt", "vllm") and self._lm_engine_impl is None:
- raise RuntimeError(
- f"Failed to initialize LM engine '{self.lm_engine}'. "
- "Check LM weights path and tokenizer availability."
- )
+ if self.lm_model is None or self.lm_tokenizer is None:
+ raise RuntimeError(f"LM engine '{self.lm_engine}' requires loaded LM model and tokenizer.")
+ common_kwargs = {"model": self.lm_model, "tokenizer": self.lm_tokenizer, "device": self.device}
+ engine_builders = {
+ "legacy": lambda: Qwen3LegacyEngine(**common_kwargs),
+ "vllm": lambda: Qwen3VllmEngine(lm_weights_path=self.lm_weights_path, **common_kwargs),
+ }
+ build_engine = engine_builders.get(self.lm_engine, None)
+ if build_engine is None:
+ raise RuntimeError(f"Unsupported LM engine '{self.lm_engine}'.")
+ self._lm_engine_impl = build_engine()
def _load_silence_latent(self):
if not self.silence_latent_path or not os.path.isfile(self.silence_latent_path):
@@ -639,34 +607,11 @@ class ACEStep15Pipeline:
stop_checker=None,
progress_label: str = "LM text",
release_vram_after: bool = True,
+ ignore_eos: bool = False,
):
- if self._lm_engine_impl is not None and hasattr(self._lm_engine_impl, "generate_text"):
- return self._lm_engine_impl.generate_text(
- prompt=prompt,
- prompt_negative=prompt_negative,
- max_tokens=max_tokens,
- temperature=temperature,
- top_p=top_p,
- top_k=top_k,
- cfg_scale=cfg_scale,
- seed=seed,
- callback=callback,
- abort_fn=self._should_abort,
- logits_processor=logits_processor,
- logits_processor_update_state=logits_processor_update_state,
- stop_checker=stop_checker,
- progress_label=progress_label,
- release_vram_after=release_vram_after,
- )
- if self.lm_engine != "legacy":
+ if self._lm_engine_impl is None:
raise RuntimeError(f"LM engine '{self.lm_engine}' is not initialized for text generation.")
- if self.lm_model is None or self.lm_tokenizer is None:
- raise RuntimeError("Legacy LM engine requires loaded LM model and tokenizer.")
- self._ensure_lm_module_loaded()
- return self._lm_legacy_text_fn(
- model=self.lm_model,
- tokenizer=self.lm_tokenizer,
- device=self.device,
+ output = self._lm_engine_impl.generate_text(
prompt=prompt,
prompt_negative=prompt_negative,
max_tokens=max_tokens,
@@ -681,8 +626,11 @@ class ACEStep15Pipeline:
logits_processor_update_state=logits_processor_update_state,
stop_checker=stop_checker,
progress_label=progress_label,
- ignore_eos=False,
+ release_vram_after=release_vram_after,
+ ignore_eos=ignore_eos,
)
+ self._lm_last_failure_reason = ""
+ return output
def _ensure_reference_phase1_modules(self):
if self._ref_metadata_processor_class is not None:
@@ -852,7 +800,9 @@ class ACEStep15Pipeline:
release_vram_after=release_vram_after,
)
if text_out is None:
- raise RuntimeError("LM phase1 failed while inferring metadata.")
+ return None, None
+ if self._should_abort():
+ return None, None
phase1_text = text_out.get("text", "") if isinstance(text_out, dict) else ""
parsed_metadata, _ = self._parse_reference_lm_output(phase1_text)
@@ -949,8 +899,8 @@ class ACEStep15Pipeline:
int(seed_value) if seed_value is not None else None,
)
- def _reserve_vllm_runtime_for_max_duration(self, tags: str, lyrics: str, negative_prompt: str, lm_cfg_scale):
- if self.lm_engine != "vllm" or self._lm_engine_impl is None or self.lm_tokenizer is None:
+ def _reserve_lm_runtime_for_max_duration(self, tags: str, lyrics: str, negative_prompt: str, lm_cfg_scale):
+ if self._lm_engine_impl is None or self.lm_tokenizer is None:
return
reserve_fn = getattr(self._lm_engine_impl, "reserve_runtime", None)
if not callable(reserve_fn):
@@ -1008,11 +958,6 @@ class ACEStep15Pipeline:
):
if cfg_scale is None:
cfg_scale = 2.5
- if self.lm_engine == "vllm" and offloadobj is not None:
- try:
- offloadobj.unload_all()
- except Exception:
- pass
metadata = {
"bpm": bpm,
@@ -1029,29 +974,11 @@ class ACEStep15Pipeline:
prompt = self._build_lm_prompt_with_cot(tags, lyrics, cot_text, is_negative_prompt=False, negative_prompt=negative_prompt)
prompt_negative = self._build_lm_prompt_with_cot(tags, lyrics, cot_text, is_negative_prompt=True, negative_prompt=negative_prompt)
- if self._lm_engine_impl is not None:
- return self._lm_engine_impl.generate_audio_codes(
- prompt=prompt,
- prompt_negative=prompt_negative,
- min_tokens=min_tokens,
- max_tokens=max_tokens,
- temperature=temperature,
- top_p=top_p,
- top_k=top_k,
- cfg_scale=cfg_scale,
- seed=seed,
- callback=callback,
- abort_fn=self._should_abort,
- )
- if self.lm_engine != "legacy":
+ if self._lm_engine_impl is None:
raise RuntimeError(f"LM engine '{self.lm_engine}' is not initialized.")
- if self.lm_model is None or self.lm_tokenizer is None:
- raise RuntimeError("Legacy LM engine requires loaded LM model and tokenizer.")
- self._ensure_lm_module_loaded()
- return self._lm_legacy_generate_fn(
- model=self.lm_model,
+ audio_codes, failure_reason = generate_audio_codes_with_engine_sampling(
+ engine=self._lm_engine_impl,
tokenizer=self.lm_tokenizer,
- device=self.device,
prompt=prompt,
prompt_negative=prompt_negative,
audio_code_mask=getattr(self, "_audio_code_mask", None),
@@ -1065,12 +992,15 @@ class ACEStep15Pipeline:
seed=seed,
callback=callback,
abort_fn=self._should_abort,
+ release_vram_after=True,
)
+ self._lm_last_failure_reason = failure_reason
+ return audio_codes
- def _release_vllm_lm_runtime(self):
- if self.lm_engine != "vllm" or self._lm_engine_impl is None:
+ def _release_lm_runtime(self):
+ if self._lm_engine_impl is None:
return
- release_fn = getattr(self._lm_engine_impl, "release_vram", None)
+ release_fn = getattr(self._lm_engine_impl, "release_runtime_allocations", None)
if not callable(release_fn):
return
try:
@@ -1233,9 +1163,9 @@ class ACEStep15Pipeline:
progress_unit="tokens",
)
if torch.is_tensor(cached):
- self._release_vllm_lm_runtime()
+ self._release_lm_runtime()
return cached.tolist()
- self._release_vllm_lm_runtime()
+ self._release_lm_runtime()
return list(cached)
def _default_timbre_latents(self, length):
@@ -1882,7 +1812,12 @@ class ACEStep15Pipeline:
elif isinstance(user_audio_codes, (list, tuple)) and user_audio_codes and isinstance(user_audio_codes[0], str):
parsed = self._parse_audio_code_string(user_audio_codes[0])
user_audio_codes = parsed if parsed else None
- keep_vllm_lm_loaded_for_phase2 = bool(self.enable_lm and self.lm_engine == "vllm" and (not has_src_audio) and (not user_audio_codes))
+ keep_lm_loaded_for_phase2 = bool(
+ self.enable_lm
+ and (not has_src_audio)
+ and (not user_audio_codes)
+ and bool(getattr(self._lm_engine_impl, "keep_loaded_for_phase2", False))
+ )
phase1_metadata = {
"bpm": user_bpm,
@@ -1890,163 +1825,169 @@ class ACEStep15Pipeline:
"timesignature": user_timesignature,
"language": user_language,
}
- if self.enable_lm:
- try:
- self._reserve_vllm_runtime_for_max_duration(tags, lyrics, lm_negative_prompt, lm_cfg_scale)
- except Exception:
- pass
- tags_for_generation = tags
- computed_phase1_metadata = {}
- refined_caption = None
- if model_mode_value == _ACE_STEP15_MODEL_MODE_DEFAULT:
- bpm = phase1_metadata["bpm"] if phase1_metadata["bpm"] is not None else _ACE_STEP15_DEFAULT_BPM
- timesignature = phase1_metadata["timesignature"] if phase1_metadata["timesignature"] is not None else _ACE_STEP15_DEFAULT_TIMESIGNATURE
- if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:
- timesignature = _ACE_STEP15_DEFAULT_TIMESIGNATURE
- keyscale = phase1_metadata["keyscale"] if phase1_metadata["keyscale"] is not None else _ACE_STEP15_DEFAULT_KEYSCALE
- keyscale = str(keyscale).strip()
- if len(keyscale) == 0:
- keyscale = _ACE_STEP15_DEFAULT_KEYSCALE
- language = phase1_metadata["language"] if phase1_metadata["language"] is not None else "en"
- else:
- if not self.enable_lm:
- raise RuntimeError("ACE-Step 1.5 model mode 1/2/3 requires an LM definition (text_encoder_URLs).")
- missing_fields = [field_name for field_name in ("bpm", "keyscale", "timesignature", "language") if phase1_metadata.get(field_name) is None]
- infer_language = phase1_metadata.get("language", None) is None
- run_phase1 = model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION) or (
- model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_MISSING and len(missing_fields) > 0
- )
- if run_phase1:
- if self.lm_engine == "vllm" and offloadobj is not None:
- try:
- offloadobj.unload_all()
- except Exception:
- pass
- phase1_seed = seed if seed is not None else 0
- _update_progress_status("LM Compute Metadata")
- phase1_metadata, refined_caption = self._run_phase1_metadata(
- tags=tags,
+ lm_runtime_retained = keep_lm_loaded_for_phase2
+ try:
+ if self.enable_lm:
+ try:
+ self._reserve_lm_runtime_for_max_duration(tags, lyrics, lm_negative_prompt, lm_cfg_scale)
+ except Exception:
+ pass
+ tags_for_generation = tags
+ computed_phase1_metadata = {}
+ refined_caption = None
+ if model_mode_value == _ACE_STEP15_MODEL_MODE_DEFAULT:
+ bpm = phase1_metadata["bpm"] if phase1_metadata["bpm"] is not None else _ACE_STEP15_DEFAULT_BPM
+ timesignature = phase1_metadata["timesignature"] if phase1_metadata["timesignature"] is not None else _ACE_STEP15_DEFAULT_TIMESIGNATURE
+ if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:
+ timesignature = _ACE_STEP15_DEFAULT_TIMESIGNATURE
+ keyscale = phase1_metadata["keyscale"] if phase1_metadata["keyscale"] is not None else _ACE_STEP15_DEFAULT_KEYSCALE
+ keyscale = str(keyscale).strip()
+ if len(keyscale) == 0:
+ keyscale = _ACE_STEP15_DEFAULT_KEYSCALE
+ language = phase1_metadata["language"] if phase1_metadata["language"] is not None else "en"
+ else:
+ if not self.enable_lm:
+ raise RuntimeError("ACE-Step 1.5 model mode 1/2/3 requires an LM definition (text_encoder_URLs).")
+ missing_fields = [field_name for field_name in ("bpm", "keyscale", "timesignature", "language") if phase1_metadata.get(field_name) is None]
+ infer_language = phase1_metadata.get("language", None) is None
+ run_phase1 = model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION) or (
+ model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_MISSING and len(missing_fields) > 0
+ )
+ if run_phase1:
+ phase1_seed = seed if seed is not None else 0
+ _update_progress_status("LM Compute Metadata")
+ phase1_metadata, refined_caption = self._run_phase1_metadata(
+ tags=tags,
+ lyrics=lyrics,
+ metadata=phase1_metadata,
+ refine_caption=model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION),
+ infer_language=infer_language,
+ seed=phase1_seed,
+ cfg_scale=1.0 if lm_cfg_scale is None else float(lm_cfg_scale),
+ negative_prompt=lm_negative_prompt,
+ temperature=temperature,
+ top_p=top_p,
+ top_k=top_k,
+ callback=_phase_callback,
+ release_vram_after=not keep_lm_loaded_for_phase2,
+ )
+ for field_name in missing_fields:
+ value = phase1_metadata.get(field_name, None) if phase1_metadata is not None else None
+ if value is not None:
+ computed_phase1_metadata[field_name] = value
+ if phase1_metadata is None:
+ return None
+ if model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION:
+ duration_value = self._parse_optional_int_custom_setting(phase1_metadata.get("duration", None))
+ if duration_value is None:
+ raise RuntimeError("LM phase1 failed to resolve required metadata 'duration'.")
+ duration_value = max(_ACE_STEP15_DURATION_MIN_SECONDS, min(_ACE_STEP15_DURATION_MAX_SECONDS, int(duration_value)))
+ duration_seconds = float(duration_value)
+ computed_phase1_metadata["duration"] = int(duration_value)
+ if len(computed_phase1_metadata) > 0:
+ print(f"[ace_step15][phase1] computed metadata: {computed_phase1_metadata}")
+ if refined_caption is not None:
+ tags_for_generation = refined_caption
+ if refined_caption.strip() != tags.strip():
+ print(f"[ace_step15][phase1] refined caption: {refined_caption}")
+ if set_header_text is not None:
+ header_parts = []
+ if len(computed_phase1_metadata) > 0:
+ computed_bits = []
+ for field_name in ("bpm", "keyscale", "timesignature", "language", "duration"):
+ if field_name in computed_phase1_metadata:
+ computed_bits.append(f"{field_name}={computed_phase1_metadata[field_name]}")
+ if len(computed_bits) > 0:
+ header_parts.append("LM computed metadata: " + "<BR>- ".join(computed_bits))
+ if refined_caption is not None and refined_caption.strip() != tags.strip():
+ header_parts.append(f"<BR>- LM refined caption: {refined_caption}")
+ if len(header_parts) > 0:
+ try:
+ set_header_text("".join(header_parts))
+ except Exception:
+ pass
+ for field_name in ("bpm", "keyscale", "timesignature", "language"):
+ if phase1_metadata.get(field_name, None) is None:
+ raise RuntimeError(f"LM phase1 failed to resolve required metadata '{field_name}'.")
+ bpm = int(phase1_metadata["bpm"])
+ timesignature = int(phase1_metadata["timesignature"])
+ if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:
+ raise RuntimeError("LM phase1 produced an unsupported timesignature.")
+ keyscale = str(phase1_metadata["keyscale"]).strip()
+ if len(keyscale) == 0:
+ raise RuntimeError("LM phase1 produced an empty keyscale.")
+ language = str(phase1_metadata["language"]).strip().lower()
+ if len(language) == 0:
+ raise RuntimeError("LM phase1 produced an empty language.")
+ if language not in _ACE_STEP15_VALID_LANGUAGE_SET:
+ raise RuntimeError(f"LM phase1 produced unsupported language '{language}'.")
+
+ duration_int = int(math.ceil(duration_seconds))
+ min_tokens = duration_int * 5
+
+ meta_cap = self._format_meta(bpm, duration_int, keyscale, timesignature)
+ use_timbre = "B" in (audio_prompt_type or "")
+
+ audio_codes = None
+ if user_audio_codes:
+ max_code = getattr(self, "_audio_code_max", None)
+ if max_code is not None:
+ bad_codes = [v for v in user_audio_codes if v < 0 or v > max_code]
+ if bad_codes:
+ raise ValueError(f"Audio codes out of range 0..{max_code}; example={bad_codes[0]}")
+ audio_codes = user_audio_codes
+ elif self.enable_lm and not has_src_audio:
+ _update_progress_status("LM Compute Audio Codes")
+ audio_codes = self._generate_audio_codes(
+ tags=tags_for_generation,
lyrics=lyrics,
- metadata=phase1_metadata,
- refine_caption=model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION),
- infer_language=infer_language,
- seed=phase1_seed,
- cfg_scale=1.0 if lm_cfg_scale is None else float(lm_cfg_scale),
- negative_prompt=lm_negative_prompt,
+ bpm=bpm,
+ duration=duration_int,
+ keyscale=keyscale,
+ timesignature=timesignature,
+ seed=seed if seed is not None else 0,
+ min_tokens=min_tokens,
+ max_tokens=min_tokens,
temperature=temperature,
top_p=top_p,
top_k=top_k,
+ language=language,
+ negative_prompt=lm_negative_prompt,
+ cfg_scale=lm_cfg_scale,
callback=_phase_callback,
- release_vram_after=not keep_vllm_lm_loaded_for_phase2,
- )
- for field_name in missing_fields:
- value = phase1_metadata.get(field_name, None) if phase1_metadata is not None else None
- if value is not None:
- computed_phase1_metadata[field_name] = value
- if phase1_metadata is None:
- return None
- if model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION:
- duration_value = self._parse_optional_int_custom_setting(phase1_metadata.get("duration", None))
- if duration_value is None:
- raise RuntimeError("LM phase1 failed to resolve required metadata 'duration'.")
- duration_value = max(_ACE_STEP15_DURATION_MIN_SECONDS, min(_ACE_STEP15_DURATION_MAX_SECONDS, int(duration_value)))
- duration_seconds = float(duration_value)
- computed_phase1_metadata["duration"] = int(duration_value)
- if len(computed_phase1_metadata) > 0:
- print(f"[ace_step15][phase1] computed metadata: {computed_phase1_metadata}")
- if refined_caption is not None:
- tags_for_generation = refined_caption
- if refined_caption.strip() != tags.strip():
- print(f"[ace_step15][phase1] refined caption: {refined_caption}")
- if set_header_text is not None:
- header_parts = []
- if len(computed_phase1_metadata) > 0:
- computed_bits = []
- for field_name in ("bpm", "keyscale", "timesignature", "language", "duration"):
- if field_name in computed_phase1_metadata:
- computed_bits.append(f"{field_name}={computed_phase1_metadata[field_name]}")
- if len(computed_bits) > 0:
- header_parts.append("LM computed metadata: " + "<BR>- ".join(computed_bits))
- if refined_caption is not None and refined_caption.strip() != tags.strip():
- header_parts.append(f"<BR>- LM refined caption: {refined_caption}")
- if len(header_parts) > 0:
- try:
- set_header_text("".join(header_parts))
- except Exception:
- pass
- for field_name in ("bpm", "keyscale", "timesignature", "language"):
- if phase1_metadata.get(field_name, None) is None:
- raise RuntimeError(f"LM phase1 failed to resolve required metadata '{field_name}'.")
- bpm = int(phase1_metadata["bpm"])
- timesignature = int(phase1_metadata["timesignature"])
- if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:
- raise RuntimeError("LM phase1 produced an unsupported timesignature.")
- keyscale = str(phase1_metadata["keyscale"]).strip()
- if len(keyscale) == 0:
- raise RuntimeError("LM phase1 produced an empty keyscale.")
- language = str(phase1_metadata["language"]).strip().lower()
- if len(language) == 0:
- raise RuntimeError("LM phase1 produced an empty language.")
- if language not in _ACE_STEP15_VALID_LANGUAGE_SET:
- raise RuntimeError(f"LM phase1 produced unsupported language '{language}'.")
-
- duration_int = int(math.ceil(duration_seconds))
- min_tokens = duration_int * 5
-
- meta_cap = self._format_meta(bpm, duration_int, keyscale, timesignature)
- use_timbre = "B" in (audio_prompt_type or "")
-
- audio_codes = None
- if user_audio_codes:
- max_code = getattr(self, "_audio_code_max", None)
- if max_code is not None:
- bad_codes = [v for v in user_audio_codes if v < 0 or v > max_code]
- if bad_codes:
- raise ValueError(f"Audio codes out of range 0..{max_code}; example={bad_codes[0]}")
- audio_codes = user_audio_codes
- elif self.enable_lm and not has_src_audio:
- _update_progress_status("LM Compute Audio Codes")
- audio_codes = self._generate_audio_codes(
- tags=tags_for_generation,
- lyrics=lyrics,
- bpm=bpm,
- duration=duration_int,
- keyscale=keyscale,
- timesignature=timesignature,
- seed=seed if seed is not None else 0,
- min_tokens=min_tokens,
- max_tokens=min_tokens,
- temperature=temperature,
- top_p=top_p,
- top_k=top_k,
- language=language,
- negative_prompt=lm_negative_prompt,
- cfg_scale=lm_cfg_scale,
- callback=_phase_callback,
- offloadobj=offloadobj,
- )
- if audio_codes is None:
- return None
- if len(audio_codes) == 0:
- failure_reason = ""
- if self._lm_engine_impl is not None:
- get_reason = getattr(self._lm_engine_impl, "get_last_failure_reason", None)
- if callable(get_reason):
- try:
- failure_reason = str(get_reason() or "")
- except Exception:
- failure_reason = ""
- extra = f" last_failure={failure_reason}" if failure_reason else ""
- raise RuntimeError(
- "Audio code generation returned 0 usable codes "
- f"(engine={self.lm_engine}, tokenizer_dir='{self.lm_tokenizer_dir}', "
- f"audio_code_vocab={len(getattr(self, '_audio_code_token_ids', []))}).{extra}"
+ offloadobj=offloadobj,
)
- max_code = getattr(self, "_audio_code_max", None)
- if audio_codes is not None and max_code is not None:
- bad_codes = [v for v in audio_codes if v < 0 or v > max_code]
- if bad_codes:
- raise RuntimeError(f"LM generated out-of-range audio codes; example={bad_codes[0]}")
+ if audio_codes is None:
+ return None
+ if self._should_abort():
+ return None
+ if len(audio_codes) == 0:
+ if self._should_abort():
+ return None
+ failure_reason = str(getattr(self, "_lm_last_failure_reason", "") or "")
+ if self._lm_engine_impl is not None:
+ get_reason = getattr(self._lm_engine_impl, "get_last_failure_reason", None)
+ if callable(get_reason):
+ try:
+ engine_reason = str(get_reason() or "")
+ if len(engine_reason) > 0:
+ failure_reason = engine_reason
+ except Exception:
+ pass
+ extra = f" last_failure={failure_reason}" if failure_reason else ""
+ raise RuntimeError(
+ "Audio code generation returned 0 usable codes "
+ f"(engine={self.lm_engine}, tokenizer_dir='{self.lm_tokenizer_dir}', "
+ f"audio_code_vocab={len(getattr(self, '_audio_code_token_ids', []))}).{extra}"
+ )
+ max_code = getattr(self, "_audio_code_max", None)
+ if audio_codes is not None and max_code is not None:
+ bad_codes = [v for v in audio_codes if v < 0 or v > max_code]
+ if bad_codes:
+ raise RuntimeError(f"LM generated out-of-range audio codes; example={bad_codes[0]}")
+ finally:
+ if lm_runtime_retained:
+ self._release_lm_runtime()
_update_progress_status("")
if audio_codes is not None:
diff --git a/models/TTS/ace_step15/qwen3_audio_codes.py b/models/TTS/ace_step15/qwen3_audio_codes.py
index f1c209b..a638c5f 100644
--- a/models/TTS/ace_step15/qwen3_audio_codes.py
+++ b/models/TTS/ace_step15/qwen3_audio_codes.py
@@ -1,10 +1,8 @@
-"""Qwen3 LM audio-code decoding engines (legacy/pt/vllm)."""
+"""Qwen3 LM audio-code sampling helpers (engine-neutral)."""
from __future__ import annotations
-import os
import re
-import sys
from typing import Any, Callable, Optional
import torch
@@ -82,40 +80,6 @@ def _postprocess_audio_codes(codes, min_tokens: int, max_tokens: int):
codes.extend([pad_val] * (min_tokens - len(codes)))
return codes[:max_tokens]
-
-def _guess_lm_model_size(model_path: str) -> str | None:
- name = (model_path or "").lower()
- if "0.6b" in name:
- return "0.6B"
- if "1.7b" in name:
- return "1.7B"
- if "4b" in name:
- return "4B"
- return None
-
-
-def _compute_vllm_gpu_utilization(model_path: str) -> float:
- try:
- if not torch.cuda.is_available():
- return 0.9
- total_gb = torch.cuda.get_device_properties(0).total_memory / (1024 ** 3)
- except Exception:
- return 0.9
- if total_gb <= 0:
- return 0.9
- target_map = {
- "0.6B": 3.0,
- "1.7B": 8.0,
- "4B": 12.0,
- }
- model_size = _guess_lm_model_size(model_path)
- target_gb = target_map.get(model_size, 3.0)
- ratio = target_gb / total_gb
- if total_gb >= 24:
- return min(0.9, max(0.2, ratio))
- return min(0.9, max(0.1, ratio))
-
-
class AudioCodeMaskProcessor(LogitsProcessor):
def __init__(self, audio_code_mask: Optional[torch.Tensor]):
self._mask_cpu = audio_code_mask
@@ -327,7 +291,7 @@ def _generate_token_ids_legacy(
return token_ids
-def generate_text_legacy(
+def generate_text_sampling(
model,
tokenizer,
device,
@@ -373,10 +337,9 @@ def generate_text_legacy(
return {"token_ids": token_ids, "text": text}
-def generate_audio_codes_legacy(
- model,
+def generate_audio_codes_with_engine_sampling(
+ engine,
tokenizer,
- device,
prompt: str,
prompt_negative: str,
audio_code_mask: Optional[torch.Tensor],
@@ -390,12 +353,10 @@ def generate_audio_codes_legacy(
seed: Optional[int],
callback=None,
abort_fn: Optional[Callable[[], bool]] = None,
+ release_vram_after: bool = True,
):
mask_processor = AudioCodeMaskProcessor(audio_code_mask)
- text_out = generate_text_legacy(
- model=model,
- tokenizer=tokenizer,
- device=device,
+ text_out = engine.generate_text(
prompt=prompt,
prompt_negative=prompt_negative,
max_tokens=max_tokens,
@@ -407,671 +368,64 @@ def generate_audio_codes_legacy(
callback=callback,
abort_fn=abort_fn,
logits_processor=mask_processor,
+ logits_processor_update_state=None,
+ stop_checker=None,
progress_label="Compute Audio Codes",
+ release_vram_after=release_vram_after,
ignore_eos=True,
)
if text_out is None:
- return None
+ return None, ""
+ if abort_fn is not None and abort_fn():
+ return None, ""
+
+ if isinstance(text_out, dict):
+ token_ids = text_out.get("token_ids", []) or []
+ decoded_text = str(text_out.get("text", "") or "")
+ else:
+ token_ids = []
+ decoded_text = str(text_out or "")
+ token_ids = [int(x) for x in token_ids]
+
+ parsed_codes = []
+ unmapped_count = 0
+ out_of_vocab_count = 0
+ vocab_size = None
+ try:
+ vocab_size = len(tokenizer.get_vocab())
+ except Exception:
+ vocab_size = None
- token_ids = text_out.get("token_ids", [])
- audio_codes = []
for token_id in token_ids:
- code_val = _token_id_to_audio_code(token_id, audio_code_token_map, tokenizer)
+ code_val = _token_id_to_audio_code(int(token_id), audio_code_token_map, tokenizer)
if code_val is not None:
- audio_codes.append(code_val)
- if len(audio_codes) >= max_tokens:
+ parsed_codes.append(code_val)
+ if len(parsed_codes) >= max_tokens:
break
+ else:
+ unmapped_count += 1
+ if vocab_size is not None and (token_id < 0 or token_id >= vocab_size):
+ out_of_vocab_count += 1
- return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)
-
-
-class Qwen3PtEngine:
- def __init__(
- self,
- model,
- tokenizer,
- device,
- audio_code_mask: Optional[torch.Tensor],
- audio_code_token_map: dict[int, int],
- ):
- self.model = model
- self.tokenizer = tokenizer
- self.device = device
- self.audio_code_mask = audio_code_mask
- self.audio_code_token_map = audio_code_token_map
- self._mask_processor = AudioCodeMaskProcessor(audio_code_mask)
-
- def generate_audio_codes(
- self,
- prompt: str,
- prompt_negative: str,
- min_tokens: int,
- max_tokens: int,
- temperature: Optional[float],
- top_p: Optional[float],
- top_k: Optional[int],
- cfg_scale: float,
- seed: Optional[int],
- callback=None,
- abort_fn: Optional[Callable[[], bool]] = None,
- ):
- if cfg_scale > 1.0:
- return generate_audio_codes_legacy(
- self.model,
- self.tokenizer,
- self.device,
- prompt,
- prompt_negative,
- self.audio_code_mask,
- self.audio_code_token_map,
- min_tokens,
- max_tokens,
- temperature,
- top_p,
- top_k,
- cfg_scale,
- seed,
- callback=callback,
- abort_fn=abort_fn,
- )
-
- if abort_fn is not None and abort_fn():
- return None
-
- inputs = self.tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
- input_ids = inputs["input_ids"].to(self.device)
- attention_mask = inputs.get("attention_mask")
- if attention_mask is not None:
- attention_mask = attention_mask.to(self.device)
-
- generator = None
- if seed is not None and seed >= 0:
- generator = torch.Generator(device=self.device)
- generator.manual_seed(seed)
-
- logits_processors = LogitsProcessorList([self._mask_processor])
-
- if callback is not None:
- callback(
- step_idx=-1,
- override_num_inference_steps=max_tokens,
- denoising_extra=f"Compute Audio Codes 0/{max_tokens}",
- progress_unit="tokens",
- )
-
- do_sample = temperature is not None and temperature > 0
- with torch.no_grad():
- outputs = self.model.generate(
- input_ids=input_ids,
- attention_mask=attention_mask,
- max_new_tokens=max_tokens,
- do_sample=do_sample,
- temperature=temperature if do_sample else 1.0,
- top_k=top_k if top_k is not None and top_k > 0 else None,
- top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,
- logits_processor=logits_processors,
- pad_token_id=self.tokenizer.pad_token_id or self.tokenizer.eos_token_id,
- generator=generator,
- )
-
- generated_ids = outputs[0][input_ids.shape[1]:]
- if generated_ids.is_cuda:
- generated_ids = generated_ids.cpu()
-
- audio_codes = []
- for token_id in generated_ids.tolist():
- code_val = _token_id_to_audio_code(int(token_id), self.audio_code_token_map, self.tokenizer)
- if code_val is not None:
- audio_codes.append(code_val)
- if len(audio_codes) >= max_tokens:
- break
-
- if callback is not None:
- callback(
- step_idx=max(0, max_tokens - 1),
- override_num_inference_steps=max_tokens,
- denoising_extra=f"Compute Audio Codes {max_tokens}/{max_tokens}",
- progress_unit="tokens",
- )
-
- return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)
-
- def generate_text(
- self,
- prompt: str,
- prompt_negative: str,
- max_tokens: int,
- temperature: Optional[float],
- top_p: Optional[float],
- top_k: Optional[int],
- cfg_scale: float,
- seed: Optional[int],
- callback=None,
- abort_fn: Optional[Callable[[], bool]] = None,
- logits_processor: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None,
- logits_processor_update_state: Optional[Callable[[int], None]] = None,
- stop_checker: Optional[Callable[[list[int], int], bool]] = None,
- progress_label: str = "LM text",
- release_vram_after: bool = True,
- ):
- del release_vram_after
- return generate_text_legacy(
- model=self.model,
- tokenizer=self.tokenizer,
- device=self.device,
- prompt=prompt,
- prompt_negative=prompt_negative,
- max_tokens=max_tokens,
- temperature=temperature,
- top_p=top_p,
- top_k=top_k,
- cfg_scale=cfg_scale,
- seed=seed,
- callback=callback,
- abort_fn=abort_fn,
- logits_processor=logits_processor,
- logits_processor_update_state=logits_processor_update_state,
- stop_checker=stop_checker,
- progress_label=progress_label,
- ignore_eos=False,
- )
-
-
-class Qwen3VllmEngine:
- def __init__(
- self,
- lm_weights_path: str,
- tokenizer,
- audio_code_mask: Optional[torch.Tensor],
- audio_code_token_map: dict[int, int],
- free_mmap_after_cuda_copy: bool = True,
- weight_load_mode: str = "lazy",
- ):
- self.lm_weights_path = lm_weights_path
- self.tokenizer = tokenizer
- self.audio_code_mask = audio_code_mask
- self.audio_code_token_map = audio_code_token_map
- self.free_mmap_after_cuda_copy = free_mmap_after_cuda_copy
- self._llm = None
- self._sampling_params_cls = None
- self._model_dir = None
- self._max_model_len_hint = None
- self._max_num_seqs_hint = None
- self._max_num_batched_tokens_hint = None
- self._last_zero_code_diagnostic = None
- self.weight_load_mode = (weight_load_mode or "lazy").strip().lower()
- if self.weight_load_mode not in ("lazy", "pinned"):
- raise ValueError(f"Unsupported vllm weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")
-
- def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):
- req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(
- prompt_len=prompt_len,
- max_tokens=max_tokens,
- cfg_scale=cfg_scale,
- )
- self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)
-
- def _release_vram(self):
- if self._llm is None:
- return
- try:
- self._llm.unload_weights()
- except Exception:
- pass
-
- def release_vram(self):
- self._release_vram()
-
- @staticmethod
- def _compute_runtime_hints(prompt_len: int, max_tokens: int, cfg_scale: float):
- max_model_len = max(8, int(prompt_len) + int(max_tokens))
- max_num_seqs = 2 if cfg_scale and cfg_scale > 1.0 else 1
- max_num_batched_tokens = max_model_len * max_num_seqs
- return max_model_len, max_num_seqs, max_num_batched_tokens
-
- def _ensure_runtime_capacity(self, max_model_len: int, max_num_seqs: int, max_num_batched_tokens: int):
- if self._max_model_len_hint is None:
- self._max_model_len_hint = max_model_len
- self._max_num_seqs_hint = max_num_seqs
- self._max_num_batched_tokens_hint = max_num_batched_tokens
- return
-
- need_grow = (
- max_model_len > int(self._max_model_len_hint)
- or max_num_seqs > int(self._max_num_seqs_hint)
- or max_num_batched_tokens > int(self._max_num_batched_tokens_hint)
- )
- if not need_grow:
- return
-
- self._max_model_len_hint = max_model_len
- self._max_num_seqs_hint = max_num_seqs
- self._max_num_batched_tokens_hint = max_num_batched_tokens
- # Existing LLM was built with smaller runtime limits; rebuild safely.
- self.close()
-
- def _ensure_nanovllm_on_path(self):
- llm_engines_root = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "shared", "llm_engines"))
- if llm_engines_root not in sys.path:
- sys.path.insert(0, llm_engines_root)
-
- def _prepare_model_dir(self):
- if self._model_dir is not None:
- return self._model_dir
-
- if not self.lm_weights_path:
- raise RuntimeError("vllm engine requires lm_weights_path")
-
- # For vLLM/nano-vllm, we can point directly at the weight file.
- # nanovllm.config will derive model_dir from the file path to find config/tokenizer.
- self._model_dir = self.lm_weights_path
- return self._model_dir
-
- def _ensure_llm(self):
- if self._llm is not None:
- return
- # Keep online-softmax path enabled for compiled LM sampling kernels.
- # Without this, Inductor may split reductions and emit a warning while
- # falling back to a slower softmax lowering.
- try:
- import torch._inductor.config as inductor_config
-
- if bool(getattr(inductor_config, "split_reductions", False)):
- inductor_config.split_reductions = False
- except Exception:
- pass
- self._ensure_nanovllm_on_path()
- try:
- from nanovllm import LLM, SamplingParams
- except Exception as exc: # pragma: no cover
- raise RuntimeError(f"nano-vllm is not available for vllm engine: {exc}") from exc
-
- model_dir = self._prepare_model_dir()
- gpu_memory_utilization = _compute_vllm_gpu_utilization(self.lm_weights_path)
- max_model_len = self._max_model_len_hint or 4096
- max_num_seqs = self._max_num_seqs_hint or 1
- max_num_batched_tokens = self._max_num_batched_tokens_hint or (max_model_len * max_num_seqs)
- self._llm = LLM(
- model=model_dir,
- enforce_eager=False,
- tensor_parallel_size=1,
- max_model_len=max_model_len,
- max_num_seqs=max_num_seqs,
- max_num_batched_tokens=max_num_batched_tokens,
- gpu_memory_utilization=gpu_memory_utilization,
- tokenizer=self.tokenizer,
- free_mmap_after_cuda_copy=self.free_mmap_after_cuda_copy,
- weight_load_mode=self.weight_load_mode,
- )
- self._sampling_params_cls = SamplingParams
-
- def close(self):
- llm = getattr(self, "_llm", None)
- self._llm = None
- if llm is not None:
- try:
- close_fn = getattr(llm, "close", None)
- if callable(close_fn):
- close_fn()
- else:
- try:
- llm.unload_weights()
- except Exception:
- pass
- try:
- llm.clear_graph_cache()
- except Exception:
- pass
- exit_fn = getattr(llm, "exit", None)
- if callable(exit_fn):
- exit_fn()
- except Exception:
- pass
- try:
- del llm
- except Exception:
- pass
- # Keep close() idempotent; this method is also used for runtime rebuilds.
- self._sampling_params_cls = None
- self._model_dir = None
-
- def get_last_failure_reason(self) -> str:
- diag = self._last_zero_code_diagnostic
- if not diag:
- return ""
- return (
- f"attempt={diag.get('attempt')} token_ids={diag.get('token_ids')} "
- f"unmapped={diag.get('unmapped')} out_of_vocab={diag.get('out_of_vocab')} "
- f"regex_hits={diag.get('regex_hits')} first_token_ids={diag.get('first_token_ids')} "
- f"text_preview='{diag.get('text_preview')}'"
- )
-
- def __del__(self):
- self.close()
-
- @staticmethod
- def _extract_text_and_tokens(output_obj) -> tuple[str, list[int]]:
- if output_obj is None:
- return "", []
- if isinstance(output_obj, dict):
- text = str(output_obj.get("text", "") or "")
- token_ids = output_obj.get("token_ids", []) or []
- return text, [int(x) for x in token_ids]
- if hasattr(output_obj, "outputs"):
- outputs = getattr(output_obj, "outputs", None)
- if outputs and len(outputs) > 0:
- text = str(getattr(outputs[0], "text", "") or "")
- token_ids = getattr(outputs[0], "token_ids", None)
- if token_ids is None:
- token_ids = getattr(outputs[0], "token_ids_list", []) or []
- return text, [int(x) for x in token_ids] if token_ids else []
- text = str(getattr(output_obj, "text", "") or "")
- token_ids = getattr(output_obj, "token_ids", None)
- if token_ids is None:
- token_ids = []
- return text, [int(x) for x in token_ids]
-
- def generate_audio_codes(
- self,
- prompt: str,
- prompt_negative: str,
- min_tokens: int,
- max_tokens: int,
- temperature: Optional[float],
- top_p: Optional[float],
- top_k: Optional[int],
- cfg_scale: float,
- seed: Optional[int],
- callback=None,
- abort_fn: Optional[Callable[[], bool]] = None,
- ):
- if abort_fn is not None and abort_fn():
- return None
-
+ regex_hits = 0
+ if len(parsed_codes) == 0 and decoded_text:
try:
- prompt_len = len(self.tokenizer.encode(prompt))
+ parsed_codes = [int(x) for x in _AUDIO_CODE_RE.findall(decoded_text)]
+ regex_hits = len(parsed_codes)
+ if len(parsed_codes) > max_tokens:
+ parsed_codes = parsed_codes[:max_tokens]
except Exception:
- prompt_len = 0
- # CFG runs conditional + unconditional sequences; runtime capacity must fit
- # the longest prompt between both or decode can enter invalid states.
- if cfg_scale > 1.0 and prompt_negative:
- try:
- prompt_len = max(prompt_len, len(self.tokenizer.encode(prompt_negative)))
- except Exception:
- pass
- req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(
- prompt_len=prompt_len,
- max_tokens=max_tokens,
- cfg_scale=cfg_scale,
- )
- self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)
- self._ensure_llm()
- if self._llm is None:
- return None
- try:
- self._llm.reset()
- except Exception:
- pass
-
- if callback is not None:
- callback(
- step_idx=-1,
- override_num_inference_steps=max_tokens,
- denoising_extra=f"Compute Audio Codes 0/{max_tokens}",
- progress_unit="tokens",
- )
-
- seed_value = None
- if seed is not None:
- try:
- seed_value = int(seed)
- except Exception:
- seed_value = None
- if seed_value is not None and seed_value < 0:
- seed_value = None
-
- temp = temperature if temperature is not None and temperature > 0 else 1e-5
- sampling_params = self._sampling_params_cls(
- temperature=temp,
- max_tokens=max_tokens,
- cfg_scale=max(cfg_scale, 1.0),
- top_k=top_k if top_k is not None and top_k > 0 else None,
- top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,
- ignore_eos=True,
- logits_processor=AudioCodeMaskProcessor(self.audio_code_mask),
- seed=seed_value,
- )
-
- audio_codes = []
- self._last_zero_code_diagnostic = None
- for attempt in range(2):
- outputs = self._llm.generate(
- prompts=[prompt],
- sampling_params=sampling_params,
- use_tqdm=True,
- unconditional_prompts=[prompt_negative] if cfg_scale > 1.0 else None,
- )
-
- token_ids = outputs[0].get("token_ids", []) if outputs else []
parsed_codes = []
- unmapped_count = 0
- out_of_vocab_count = 0
- vocab_size = None
- try:
- vocab_size = len(self.tokenizer.get_vocab())
- except Exception:
- vocab_size = None
- for token_id in token_ids:
- code_val = _token_id_to_audio_code(int(token_id), self.audio_code_token_map, self.tokenizer)
- if code_val is not None:
- parsed_codes.append(code_val)
- if len(parsed_codes) >= max_tokens:
- break
- else:
- unmapped_count += 1
- if vocab_size is not None and (token_id < 0 or token_id >= vocab_size):
- out_of_vocab_count += 1
-
- # Fallback parser: extract <|audio_code_x|> from decoded text if token_ids couldn't be mapped.
- if len(parsed_codes) == 0 and outputs:
- try:
- decoded_text = str(outputs[0].get("text", ""))
- parsed_codes = [int(x) for x in _AUDIO_CODE_RE.findall(decoded_text)]
- if len(parsed_codes) > max_tokens:
- parsed_codes = parsed_codes[:max_tokens]
- except Exception:
- parsed_codes = []
-
- if parsed_codes:
- audio_codes = parsed_codes
- self._last_zero_code_diagnostic = None
- break
-
- decoded_text = ""
regex_hits = 0
- if outputs:
- try:
- decoded_text = str(outputs[0].get("text", ""))
- except Exception:
- decoded_text = ""
- if decoded_text:
- try:
- regex_hits = len(_AUDIO_CODE_RE.findall(decoded_text))
- except Exception:
- regex_hits = 0
- text_preview = decoded_text[:280].replace("\n", "\\n")
- self._last_zero_code_diagnostic = {
- "attempt": attempt + 1,
- "token_ids": len(token_ids),
- "unmapped": unmapped_count,
- "out_of_vocab": out_of_vocab_count,
- "regex_hits": regex_hits,
- "first_token_ids": token_ids[:24],
- "text_preview": text_preview,
- }
- print(
- f"[ace_step15][vllm][diagnostic] attempt={attempt+1} "
- f"token_ids={len(token_ids)} parsed_codes=0 "
- f"unmapped={unmapped_count} out_of_vocab={out_of_vocab_count} "
- f"first_token_ids={token_ids[:24]} text_preview='{text_preview}'"
- )
-
- if attempt == 0:
- print("[ace_step15] Warning: vllm LM returned 0 audio codes; rebuilding LM runtime and retrying once.")
- self.close()
- self._ensure_llm()
- if self._llm is None:
- break
-
- if callback is not None:
- callback(
- step_idx=max(0, max_tokens - 1),
- override_num_inference_steps=max_tokens,
- denoising_extra=f"Compute Audio Codes {max_tokens}/{max_tokens}",
- progress_unit="tokens",
- )
-
- self._release_vram()
-
- return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)
-
- def generate_text(
- self,
- prompt: str,
- prompt_negative: str,
- max_tokens: int,
- temperature: Optional[float],
- top_p: Optional[float],
- top_k: Optional[int],
- cfg_scale: float,
- seed: Optional[int],
- callback=None,
- abort_fn: Optional[Callable[[], bool]] = None,
- logits_processor: Optional[Any] = None,
- logits_processor_update_state: Optional[Callable[[int], None]] = None,
- stop_checker: Optional[Callable[[list[int], int], bool]] = None,
- progress_label: str = "LM text",
- release_vram_after: bool = True,
- ):
- del stop_checker # nanovllm does not support external stop callbacks yet.
- if abort_fn is not None and abort_fn():
- return None
-
- try:
- prompt_len = len(self.tokenizer.encode(prompt))
- except Exception:
- prompt_len = 0
- if cfg_scale > 1.0 and prompt_negative:
- try:
- prompt_len = max(prompt_len, len(self.tokenizer.encode(prompt_negative)))
- except Exception:
- pass
-
- req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(
- prompt_len=prompt_len,
- max_tokens=max_tokens,
- cfg_scale=cfg_scale,
- )
- self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)
- self._ensure_llm()
- if self._llm is None:
- return None
- try:
- self._llm.reset()
- except Exception:
- pass
-
- if callback is not None:
- callback(
- step_idx=-1,
- override_num_inference_steps=max_tokens,
- denoising_extra=f"{progress_label} 0/{max_tokens}",
- progress_unit="tokens",
- )
-
- seed_value = None
- if seed is not None:
- try:
- seed_value = int(seed)
- except Exception:
- seed_value = None
- if seed_value is not None and seed_value < 0:
- seed_value = None
-
- temp = temperature if temperature is not None and temperature > 0 else 1e-5
- sampling_params = self._sampling_params_cls(
- temperature=temp,
- max_tokens=max_tokens,
- cfg_scale=max(cfg_scale, 1.0),
- top_k=top_k if top_k is not None and top_k > 0 else None,
- top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,
- ignore_eos=False,
- logits_processor=logits_processor,
- logits_processor_update_state=logits_processor_update_state,
- seed=seed_value,
- )
-
- outputs = self._llm.generate(
- prompts=[prompt],
- sampling_params=sampling_params,
- use_tqdm=True,
- unconditional_prompts=[prompt_negative] if cfg_scale > 1.0 else None,
+ if abort_fn is not None and abort_fn():
+ return None, ""
+
+ failure_reason = ""
+ if len(parsed_codes) == 0:
+ text_preview = decoded_text[:280].replace("\n", "\\n")
+ failure_reason = (
+ f"token_ids={len(token_ids)} unmapped={unmapped_count} out_of_vocab={out_of_vocab_count} "
+ f"regex_hits={regex_hits} first_token_ids={token_ids[:24]} text_preview='{text_preview}'"
)
- text = ""
- token_ids: list[int] = []
- if outputs:
- text, token_ids = self._extract_text_and_tokens(outputs[0])
- if (not text) and token_ids:
- try:
- text = self.tokenizer.decode(token_ids, skip_special_tokens=False)
- except Exception:
- text = ""
-
- if callback is not None:
- callback(
- step_idx=max(0, max_tokens - 1),
- override_num_inference_steps=max_tokens,
- denoising_extra=f"{progress_label} {max_tokens}/{max_tokens}",
- progress_unit="tokens",
- )
-
- if release_vram_after:
- self._release_vram()
-
- return {"token_ids": token_ids, "text": text}
-
-def create_qwen3_lm_engine(
- engine_name: str,
- model,
- tokenizer,
- device,
- lm_weights_path: str,
- audio_code_mask: Optional[torch.Tensor],
- audio_code_token_map: dict[int, int],
- free_mmap_after_cuda_copy: bool = True,
- weight_load_mode: str = "lazy",
-):
- engine = (engine_name or "legacy").strip().lower()
- if engine == "vllm":
- if not lm_weights_path or tokenizer is None:
- return None
- return Qwen3VllmEngine(
- lm_weights_path=lm_weights_path,
- tokenizer=tokenizer,
- audio_code_mask=audio_code_mask,
- audio_code_token_map=audio_code_token_map,
- free_mmap_after_cuda_copy=free_mmap_after_cuda_copy,
- weight_load_mode=weight_load_mode,
- )
- if engine == "pt":
- if model is None or tokenizer is None:
- return None
- return Qwen3PtEngine(
- model=model,
- tokenizer=tokenizer,
- device=device,
- audio_code_mask=audio_code_mask,
- audio_code_token_map=audio_code_token_map,
- )
- return None
+ return _postprocess_audio_codes(parsed_codes, min_tokens, max_tokens), failure_reason
diff --git a/models/TTS/ace_step15/qwen3_lm_engines.py b/models/TTS/ace_step15/qwen3_lm_engines.py
new file mode 100644
index 0000000..8d98c3a
--- /dev/null
+++ b/models/TTS/ace_step15/qwen3_lm_engines.py
@@ -0,0 +1,158 @@
+"""Qwen3 LM engine adapters."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Protocol, runtime_checkable
+
+from shared.llm_engines.nanovllm.vllm_support import NanoVllmTextEngine
+
+from .qwen3_audio_codes import generate_text_sampling
+
+
+@runtime_checkable
+class Qwen3LmEngine(Protocol):
+ keep_loaded_for_phase2: bool
+
+ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):
+ ...
+
+ def release_runtime_allocations(self):
+ ...
+
+ def get_last_failure_reason(self) -> str:
+ ...
+
+ def generate_text(
+ self,
+ prompt: str,
+ prompt_negative: str,
+ max_tokens: int,
+ temperature: Optional[float],
+ top_p: Optional[float],
+ top_k: Optional[int],
+ cfg_scale: float,
+ seed: Optional[int],
+ callback=None,
+ abort_fn: Optional[Callable[[], bool]] = None,
+ logits_processor: Optional[Callable[[Any, Any], Any]] = None,
+ logits_processor_update_state: Optional[Callable[[int], None]] = None,
+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,
+ progress_label: str = "LM text",
+ release_vram_after: bool = True,
+ ignore_eos: bool = False,
+ ):
+ ...
+
+
+class Qwen3LegacyEngine:
+ keep_loaded_for_phase2 = False
+
+ def __init__(self, model, tokenizer, device):
+ self.model = model
+ self.tokenizer = tokenizer
+ self.device = device
+
+ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):
+ del prompt_len, max_tokens, cfg_scale
+
+ def release_runtime_allocations(self):
+ return
+
+ def get_last_failure_reason(self) -> str:
+ return ""
+
+ def generate_text(
+ self,
+ prompt: str,
+ prompt_negative: str,
+ max_tokens: int,
+ temperature: Optional[float],
+ top_p: Optional[float],
+ top_k: Optional[int],
+ cfg_scale: float,
+ seed: Optional[int],
+ callback=None,
+ abort_fn: Optional[Callable[[], bool]] = None,
+ logits_processor: Optional[Callable[[Any, Any], Any]] = None,
+ logits_processor_update_state: Optional[Callable[[int], None]] = None,
+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,
+ progress_label: str = "LM text",
+ release_vram_after: bool = True,
+ ignore_eos: bool = False,
+ ):
+ del release_vram_after
+ return generate_text_sampling(
+ model=self.model,
+ tokenizer=self.tokenizer,
+ device=self.device,
+ prompt=prompt,
+ prompt_negative=prompt_negative,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ top_p=top_p,
+ top_k=top_k,
+ cfg_scale=cfg_scale,
+ seed=seed,
+ callback=callback,
+ abort_fn=abort_fn,
+ logits_processor=logits_processor,
+ logits_processor_update_state=logits_processor_update_state,
+ stop_checker=stop_checker,
+ progress_label=progress_label,
+ ignore_eos=ignore_eos,
+ )
+
+
+class Qwen3VllmEngine(NanoVllmTextEngine):
+ keep_loaded_for_phase2 = True
+
+ def __init__(self, model, tokenizer, device, lm_weights_path: str):
+ del device
+ super().__init__(model=model, model_path=lm_weights_path, tokenizer=tokenizer)
+
+ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):
+ return super().reserve_runtime(prompt_len=prompt_len, max_tokens=max_tokens, cfg_scale=cfg_scale)
+
+ def release_runtime_allocations(self):
+ return super().release_runtime_allocations()
+
+ def get_last_failure_reason(self) -> str:
+ return super().get_last_failure_reason()
+
+ def generate_text(
+ self,
+ prompt: str,
+ prompt_negative: str,
+ max_tokens: int,
+ temperature: Optional[float],
+ top_p: Optional[float],
+ top_k: Optional[int],
+ cfg_scale: float,
+ seed: Optional[int],
+ callback=None,
+ abort_fn: Optional[Callable[[], bool]] = None,
+ logits_processor: Optional[Callable[[Any, Any], Any]] = None,
+ logits_processor_update_state: Optional[Callable[[int], None]] = None,
+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,
+ progress_label: str = "LM text",
+ release_vram_after: bool = True,
+ ignore_eos: bool = False,
+ ):
+ return super().generate_text(
+ prompt=prompt,
+ prompt_negative=prompt_negative,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ top_p=top_p,
+ top_k=top_k,
+ cfg_scale=cfg_scale,
+ seed=seed,
+ callback=callback,
+ abort_fn=abort_fn,
+ logits_processor=logits_processor,
+ logits_processor_update_state=logits_processor_update_state,
+ stop_checker=stop_checker,
+ progress_label=progress_label,
+ release_vram_after=release_vram_after,
+ ignore_eos=ignore_eos,
+ )
diff --git a/models/TTS/ace_step_handler.py b/models/TTS/ace_step_handler.py
index 57905cc..7557469 100644
--- a/models/TTS/ace_step_handler.py
+++ b/models/TTS/ace_step_handler.py
@@ -490,7 +490,6 @@ class family_handler:
lm_weights = text_encoder_filename
lm_tokenizer_dir = _get_model_path(model_def, "ace_step15_lm_tokenizer_dir", _ace_step15_lm_ckpt_dir(lm_folder))
silence_latent = _get_model_path(model_def, "ace_step15_silence_latent", _ace_step15_ckpt_file(ACE_STEP15_SILENCE_LATENT_NAME))
- lm_vllm_weight_mode = _get_model_path(model_def, "ace_step15_vllm_weight_mode", "lazy")
if enable_lm:
lm_weight_name = os.path.basename(str(lm_weights)) if lm_weights else ""
print(f"[ace_step15] LM engine='{lm_decoder_engine}' | LM weights='{lm_weight_name}'")
@@ -508,7 +507,6 @@ class family_handler:
enable_lm=enable_lm,
ignore_lm_cache_seed=ignore_lm_cache_seed,
lm_decoder_engine=lm_decoder_engine,
- lm_vllm_weight_mode=lm_vllm_weight_mode,
dtype=dtype or torch.bfloat16,
)
@@ -517,9 +515,11 @@ class family_handler:
"text_encoder_2": pipeline.text_encoder_2,
"codec": pipeline.audio_vae,
}
- if lm_decoder_engine != "vllm" and text_encoder_filename and pipeline.lm_model is not None:
+ if text_encoder_filename and pipeline.lm_model is not None:
pipe["text_encoder"] = pipeline.lm_model
+ pipe = { "pipe": pipe, "coTenantsMap": {}, }
+
if save_quantized and transformer_weights:
from wgp import save_quantized_model
diff --git a/shared/llm_engines/nanovllm/config.py b/shared/llm_engines/nanovllm/config.py
index c39c3ea..55329ec 100644
--- a/shared/llm_engines/nanovllm/config.py
+++ b/shared/llm_engines/nanovllm/config.py
@@ -12,9 +12,7 @@ class Config:
gpu_memory_utilization: float = 0.9
tensor_parallel_size: int = 1
enforce_eager: bool = False
- weight_load_mode: str = "lazy" # lazy | pinned
hf_config: AutoConfig | None = None
- free_mmap_after_cuda_copy: bool = True
eos: int = -1
kvcache_block_size: int = 256
num_kvcache_blocks: int = -1
@@ -30,8 +28,6 @@ class Config:
self.model_dir = self.model
assert self.kvcache_block_size % 256 == 0
assert 1 <= self.tensor_parallel_size <= 8
- if self.weight_load_mode not in ("lazy", "pinned"):
- raise ValueError(f"Unsupported weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")
self.hf_config = AutoConfig.from_pretrained(self.model_dir)
self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings)
assert self.max_num_batched_tokens >= self.max_model_len
diff --git a/shared/llm_engines/nanovllm/engine/block_manager.py b/shared/llm_engines/nanovllm/engine/block_manager.py
index 4954044..640b541 100644
--- a/shared/llm_engines/nanovllm/engine/block_manager.py
+++ b/shared/llm_engines/nanovllm/engine/block_manager.py
@@ -2,7 +2,7 @@ from collections import deque
import xxhash
import numpy as np
-from nanovllm.engine.sequence import Sequence
+from .sequence import Sequence
class Block:
diff --git a/shared/llm_engines/nanovllm/engine/llm_engine.py b/shared/llm_engines/nanovllm/engine/llm_engine.py
index 6edec65..8c42fdf 100644
--- a/shared/llm_engines/nanovllm/engine/llm_engine.py
+++ b/shared/llm_engines/nanovllm/engine/llm_engine.py
@@ -5,17 +5,18 @@ from tqdm.auto import tqdm
from transformers import AutoTokenizer
import torch.multiprocessing as mp
-from nanovllm.config import Config
-from nanovllm.sampling_params import SamplingParams
-from nanovllm.engine.sequence import Sequence
-from nanovllm.engine.scheduler import Scheduler
-from nanovllm.engine.block_manager import BlockManager
-from nanovllm.engine.model_runner import ModelRunner
+from ..config import Config
+from ..sampling_params import SamplingParams
+from .sequence import Sequence
+from .scheduler import Scheduler
+from .block_manager import BlockManager
+from .model_runner import ModelRunner
class LLMEngine:
def __init__(self, model, **kwargs):
+ model_object = kwargs.get("model_object", None)
config_fields = {field.name for field in fields(Config)}
config_kwargs = {k: v for k, v in kwargs.items() if k in config_fields}
config = Config(model, **config_kwargs)
@@ -29,7 +30,7 @@ class LLMEngine:
process.start()
self.ps.append(process)
self.events.append(event)
- self.model_runner = ModelRunner(config, 0, self.events)
+ self.model_runner = ModelRunner(config, 0, self.events, model_object=model_object)
tokenizer = kwargs.get("tokenizer", None)
if tokenizer is not None:
self.tokenizer = tokenizer
@@ -71,7 +72,7 @@ class LLMEngine:
return
self._closed = True
try:
- self.unload_weights()
+ self.reset_runtime_state()
except Exception:
pass
try:
@@ -98,12 +99,12 @@ class LLMEngine:
except Exception:
pass
- def unload_weights(self):
+ def reset_runtime_state(self):
runner = getattr(self, "model_runner", None)
if runner is None:
return
- runner.unload_weights()
- # KV cache is invalid after unload/reload, so cached prefix block metadata
+ runner.reset_runtime_state()
+ # KV cache is invalid after runtime reset/reprepare, so cached prefix block metadata
# must be dropped as well to prevent stale-cache reuse.
try:
self.reset()
@@ -124,18 +125,6 @@ class LLMEngine:
return
runner.clear_graph_cache()
- def reset_guard_counts(self):
- runner = getattr(self, "model_runner", None)
- if runner is None:
- return
- runner.call("reset_guard_counts")
-
- def get_guard_counts(self, reset: bool = False):
- runner = getattr(self, "model_runner", None)
- if runner is None:
- return {}
- return runner.call("get_guard_counts", reset)
-
def add_request(self, prompt: str | list[int], sampling_params: SamplingParams, unconditional_prompt: str | list[int] | None = None):
if isinstance(prompt, str):
prompt = self.tokenizer.encode(prompt)
@@ -208,8 +197,8 @@ class LLMEngine:
) -> list[str]:
if self.scheduler is None:
raise RuntimeError("LLM engine is closed.")
- # Ensure weights/KV cache are ready for lazy/pinned modes, and sync scheduler blocks.
- self.model_runner.ensure_weights_loaded()
+ # Ensure model runtime/KV cache are prepared, and sync scheduler blocks.
+ self.model_runner.ensure_runtime_ready()
if (self.config.num_kvcache_blocks > 0 and
len(self.scheduler.block_manager.blocks) != self.config.num_kvcache_blocks):
self.scheduler.block_manager = BlockManager(
diff --git a/shared/llm_engines/nanovllm/engine/model_runner.py b/shared/llm_engines/nanovllm/engine/model_runner.py
index 9b765ee..aa1e12a 100644
--- a/shared/llm_engines/nanovllm/engine/model_runner.py
+++ b/shared/llm_engines/nanovllm/engine/model_runner.py
@@ -6,12 +6,10 @@ from multiprocessing.synchronize import Event
from multiprocessing.shared_memory import SharedMemory
import sys
-from nanovllm.config import Config
-from nanovllm.engine.sequence import Sequence
-from nanovllm.models import resolve_model_class
-from nanovllm.layers.sampler import Sampler
-from nanovllm.utils.context import set_context, get_context, reset_context
-from nanovllm.utils.loader import load_model, WeightStore
+from ..config import Config
+from .sequence import Sequence
+from ..layers.sampler import Sampler
+from ..utils.context import set_context, get_context, reset_context
import socket
@@ -44,7 +42,7 @@ def find_available_port(start_port: int = 2333, max_attempts: int = 100) -> int:
class ModelRunner:
- def __init__(self, config: Config, rank: int, event: Event | list[Event]):
+ def __init__(self, config: Config, rank: int, event: Event | list[Event], model_object=None):
# Enable capturing scalar outputs to avoid graph breaks from Tensor.item() calls
torch._dynamo.config.capture_scalar_outputs = True
@@ -91,12 +89,7 @@ class ModelRunner:
config_dtype = torch.bfloat16
self.dtype = config_dtype # Save for later use
- self.weight_load_mode = (config.weight_load_mode or "lazy").lower()
- self._weights_loaded = False
- self._weight_store = None
- self._is_quanto_int8 = False
- self._model_cls = resolve_model_class(hf_config)
- self._model_is_meta = False
+ self._runtime_ready = False
self._graph_cache = {}
self._graph_cache_order = []
self._logits_bias_cache = {}
@@ -104,7 +97,12 @@ class ModelRunner:
self._guard_counts = {}
self._guard_seen_details = set()
torch.set_default_dtype(config_dtype)
- self.model = self._build_deferred_model(hf_config)
+ if model_object is None:
+ raise RuntimeError(
+ "nanovllm now requires a preloaded MMGP model object. "
+ "Pass model_object=... when creating LLM."
+ )
+ self.model = model_object
self.sampler = Sampler()
# Pre-allocate buffers for sampling (optimization: avoid repeated tensor creation)
@@ -123,78 +121,16 @@ class ModelRunner:
self.shm = SharedMemory(name="nanovllm")
self.loop()
- def _build_deferred_model(self, hf_config):
- # Lazy/pinned mode should not allocate full CPU parameter storage upfront.
- with torch.device("meta"):
- model = self._model_cls(hf_config)
- self._model_is_meta = True
- return model
-
- def ensure_weights_loaded(self):
- if self._weights_loaded:
- return
- print(f"Loading LM Weights '{self.config.model}' in vllm...")
- default_dtype = torch.get_default_dtype()
- torch.set_default_dtype(self.dtype)
- try:
- self._ensure_weight_store()
- if self.model is None:
- self.model = self._build_deferred_model(self.config.hf_config)
- self._tie_word_embeddings_if_needed()
- self._mark_tied_meta_parameter_ids()
- # Tie before materialization so lm_head does not allocate a duplicate tensor.
- was_meta_model = bool(self._model_is_meta)
- if self._is_quanto_int8:
- for module in self.model.modules():
- prepare = getattr(module, "prepare_for_quantized_load", None)
- if callable(prepare):
- prepare()
- # Keep model meta until each parameter is replaced directly from safetensor
- # to avoid allocating a dense, fully materialized CUDA shell first.
- load_model(
- self.model,
- "",
- weight_store=self._weight_store,
- # We keep tensors tied to the active weight reader only within each
- # load iteration; explicit cloning is not needed and avoids a full
- # model-sized temporary copy in RAM.
- clone_loaded_tensors=False,
- )
- if was_meta_model:
- self._materialize_runtime_buffers()
- self._model_is_meta = False
- self._close_weight_store_if_lazy()
- self._weights_loaded = True
- self.allocate_kv_cache()
- if not self.enforce_eager:
- self.capture_cudagraph()
- finally:
- if not self._weights_loaded:
- self._close_weight_store_if_lazy()
- torch.set_default_device("cpu")
- torch.set_default_dtype(default_dtype)
-
- def _ensure_weight_store(self):
- if self._weight_store is not None:
- return
- self._weight_store = WeightStore(self.config.model_file or self.config.model_dir, mode=self.weight_load_mode)
- self._is_quanto_int8 = bool(getattr(self._weight_store, "is_quanto_int8", False))
-
- def _close_weight_store_if_lazy(self):
- if self._weight_store is None:
+ def ensure_runtime_ready(self):
+ if self._runtime_ready:
return
- close_fn = getattr(self._weight_store, "close", None)
- if callable(close_fn):
- close_fn()
- self._weight_store = None
-
- def _materialize_runtime_buffers(self):
if self.model is None:
- return
- for module in self.model.modules():
- materialize = getattr(module, "materialize_cache", None)
- if callable(materialize):
- materialize()
+ raise RuntimeError("LLM model object is not available.")
+ self._tie_word_embeddings_if_needed()
+ self.allocate_kv_cache()
+ if not self.enforce_eager:
+ self.capture_cudagraph()
+ self._runtime_ready = True
def _get_tied_embeddings(self):
if self.model is None:
@@ -212,17 +148,6 @@ class ModelRunner:
return None
return lm_head, embed
- def _mark_tied_meta_parameter_ids(self):
- tied = self._get_tied_embeddings()
- if tied is None:
- self.model._vllm_tied_param_ids = set()
- return
- lm_head, embed = tied
- if lm_head.weight is not embed.weight:
- self.model._vllm_tied_param_ids = set()
- return
- self.model._vllm_tied_param_ids = {id(embed.weight)}
-
def _tie_word_embeddings_if_needed(self):
tied = self._get_tied_embeddings()
if tied is None:
@@ -233,8 +158,8 @@ class ModelRunner:
if lm_head.weight is not embed.weight:
raise RuntimeError("Failed to retie lm_head.weight with embed_tokens.weight.")
- def unload_weights(self):
- if not self._weights_loaded:
+ def reset_runtime_state(self):
+ if not self._runtime_ready:
return
# Clear attention KV cache refs so we don't write into freed storage later.
try:
@@ -249,8 +174,8 @@ class ModelRunner:
del self.kv_cache
except Exception:
pass
- # CUDA graphs captured against previous weight/KV pointers are unsafe after unload/reload.
- # Force recapture on next load to avoid stale-pointer illegal memory access.
+ # CUDA graphs captured against previous model/KV pointers are unsafe after runtime reset.
+ # Force recapture on next prepare to avoid stale-pointer illegal memory access.
try:
self.clear_graph_cache()
except Exception:
@@ -268,17 +193,7 @@ class ModelRunner:
pass
self._logits_bias_cache.clear()
self._sampling_generator = None
- self._weights_loaded = False
- self.model = None
- self._model_is_meta = False
- if self._weight_store is not None:
- try:
- close_fn = getattr(self._weight_store, "close", None)
- if callable(close_fn):
- close_fn()
- except Exception:
- pass
- self._weight_store = None
+ self._runtime_ready = False
gc.collect()
def _get_graph_capture_signature(self):
@@ -336,16 +251,6 @@ class ModelRunner:
if count == 1:
print(f"[nanovllm][guard] {name}")
- def reset_guard_counts(self):
- self._guard_counts.clear()
- self._guard_seen_details.clear()
-
- def get_guard_counts(self, reset: bool = False):
- counts = dict(self._guard_counts)
- if reset:
- self.reset_guard_counts()
- return counts
-
def _get_logits_bias(self, seq: Sequence, logits: torch.Tensor):
bias = getattr(seq, "logits_bias", None)
if bias is None or not torch.is_tensor(bias):
@@ -430,17 +335,9 @@ class ModelRunner:
def exit(self):
try:
- self.unload_weights()
- except Exception:
- pass
- try:
- if self._weight_store is not None:
- close_fn = getattr(self._weight_store, "close", None)
- if callable(close_fn):
- close_fn()
+ self.reset_runtime_state()
except Exception:
pass
- self._weight_store = None
self._release_sample_buffers()
self._logits_bias_cache.clear()
self._guard_counts.clear()
@@ -449,7 +346,6 @@ class ModelRunner:
self.sampler = None
if hasattr(self, "model"):
self.model = None
- self._model_is_meta = False
if self.world_size > 1:
self.shm.close()
dist.barrier()
@@ -710,7 +606,7 @@ class ModelRunner:
"""Run model forward and sampling. For CFG sequences, batch is structured as:
[cond_seq1, cond_seq2, ..., uncond_seq1, uncond_seq2, ...]
where uncond_seqi is the paired unconditional sequence of cond_seqi."""
- self.ensure_weights_loaded()
+ self.ensure_runtime_ready()
# Check if this is a CFG batch (contains paired conditional and unconditional sequences)
is_cfg_batch = seqs[0].cfg_scale > 1.0 and seqs[0].paired_seq is not None
if is_cfg_batch:
@@ -878,7 +774,7 @@ class ModelRunner:
def capture_cudagraph(self):
config = self.config
cache_key = (config.max_model_len, config.max_num_seqs)
- model_device = self._get_model_device()
+ model_device = torch.device("cuda") if torch.cuda.is_available() else self._get_model_device()
cached = self._graph_cache.get(cache_key)
if cached is not None:
current_sig = self._get_graph_capture_signature()
diff --git a/shared/llm_engines/nanovllm/engine/scheduler.py b/shared/llm_engines/nanovllm/engine/scheduler.py
index 3178c22..60162a9 100644
--- a/shared/llm_engines/nanovllm/engine/scheduler.py
+++ b/shared/llm_engines/nanovllm/engine/scheduler.py
@@ -1,8 +1,8 @@
from collections import deque
-from nanovllm.config import Config
-from nanovllm.engine.sequence import Sequence, SequenceStatus
-from nanovllm.engine.block_manager import BlockManager
+from ..config import Config
+from .sequence import Sequence, SequenceStatus
+from .block_manager import BlockManager
class Scheduler:
diff --git a/shared/llm_engines/nanovllm/engine/sequence.py b/shared/llm_engines/nanovllm/engine/sequence.py
index bda2e53..86869d5 100644
--- a/shared/llm_engines/nanovllm/engine/sequence.py
+++ b/shared/llm_engines/nanovllm/engine/sequence.py
@@ -3,7 +3,7 @@ from enum import Enum, auto
from itertools import count
from typing import Optional, Callable, Any
-from nanovllm.sampling_params import SamplingParams
+from ..sampling_params import SamplingParams
class SequenceStatus(Enum):
diff --git a/shared/llm_engines/nanovllm/layers/attention.py b/shared/llm_engines/nanovllm/layers/attention.py
index e416139..e03f43c 100644
--- a/shared/llm_engines/nanovllm/layers/attention.py
+++ b/shared/llm_engines/nanovllm/layers/attention.py
@@ -4,7 +4,7 @@ import triton
import triton.language as tl
from flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
-from nanovllm.utils.context import get_context
+from ..utils.context import get_context
@triton.jit
diff --git a/shared/llm_engines/nanovllm/layers/embed_head.py b/shared/llm_engines/nanovllm/layers/embed_head.py
index 4475b41..793063a 100644
--- a/shared/llm_engines/nanovllm/layers/embed_head.py
+++ b/shared/llm_engines/nanovllm/layers/embed_head.py
@@ -3,7 +3,7 @@ from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
-from nanovllm.utils.context import get_context
+from ..utils.context import get_context
def _get_tp_info():
diff --git a/shared/llm_engines/nanovllm/layers/linear.py b/shared/llm_engines/nanovllm/layers/linear.py
index ab19766..b503342 100644
--- a/shared/llm_engines/nanovllm/layers/linear.py
+++ b/shared/llm_engines/nanovllm/layers/linear.py
@@ -2,27 +2,6 @@ import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
-import math
-
-try:
- from shared.kernels import quanto_int8_triton as _shared_quanto_int8_triton
-except Exception: # pragma: no cover
- _shared_quanto_int8_triton = None # type: ignore
-
-
-def _shared_kernel_available() -> bool:
- if _shared_quanto_int8_triton is None:
- return False
- is_available = getattr(_shared_quanto_int8_triton, "is_available", None)
- if not callable(is_available):
- return False
- try:
- return bool(is_available())
- except Exception:
- return False
-
-
-_TRITON_AVAILABLE = _shared_kernel_available()
def divide(numerator, denominator):
@@ -36,232 +15,15 @@ def _get_tp_info():
return 0, 1
-def _flatten_scale(scale: torch.Tensor) -> torch.Tensor:
- if scale.ndim == 2 and scale.shape[1] == 1:
- return scale.view(-1)
- if scale.ndim == 1:
- return scale
- return scale.reshape(-1)
-
-
-def _run_triton_fused_int8_mm(
- x2d: torch.Tensor,
- qweight_t: torch.Tensor,
- qweight_scale_fp32: torch.Tensor,
- input_scale: float | None = None,
-) -> torch.Tensor:
- del input_scale
- if _shared_quanto_int8_triton is None:
- raise RuntimeError("shared.kernels.quanto_int8_triton is unavailable")
- run_kernel = getattr(_shared_quanto_int8_triton, "fused_quant_scaled_mm_transposed", None)
- if run_kernel is None:
- raise RuntimeError("shared.kernels.quanto_int8_triton.fused_quant_scaled_mm_transposed is missing")
-
- m, k = x2d.shape
- k2, n = qweight_t.shape
- if k != k2:
- raise RuntimeError(f"Triton int8 GEMM shape mismatch: x={x2d.shape}, w_t={qweight_t.shape}")
- qweight_scale_fp32 = _flatten_scale(qweight_scale_fp32)
- if qweight_scale_fp32.numel() != n:
- raise RuntimeError(
- f"Triton int8 qweight_scale length mismatch: expected {n}, got {qweight_scale_fp32.numel()}"
- )
- return run_kernel(x2d, qweight_t, qweight_scale_fp32, out_dtype=x2d.dtype)
-
+class LinearBase(nn.Linear):
-class LinearBase(nn.Module):
-
- def __init__(
- self,
- input_size: int,
- output_size: int,
- bias: bool = False,
- tp_dim: int | None = None,
- ):
- super().__init__()
- self.tp_dim = tp_dim
+ def __init__(self, input_size: int, output_size: int, bias: bool = False):
+ super().__init__(input_size, output_size, bias=bias)
self.tp_rank, self.tp_size = _get_tp_info()
- self.input_size = input_size
- self.output_size = output_size
- self.weight = nn.Parameter(torch.empty(output_size, input_size))
- self.weight.weight_loader = self.weight_loader
- self.register_buffer("qweight_data", torch.empty(0, dtype=torch.int8))
- self.register_buffer("qweight_t", torch.empty(0, dtype=torch.int8))
- self.register_buffer("qweight_scale", torch.empty(0))
- self.register_buffer("qweight_scale_fp32", torch.empty(0, dtype=torch.float32))
- self.register_buffer("input_scale", torch.ones((), dtype=torch.bfloat16))
- self.register_buffer("output_scale", torch.ones((), dtype=torch.bfloat16))
- self.use_int8_weight = False
- self.use_triton_int8 = False
- self._input_scale_value = 1.0
- self._quant_expected_shards = 1
- self._quant_data_loaded = set()
- self._quant_scale_loaded = set()
- if bias:
- self.bias = nn.Parameter(torch.empty(output_size))
- self.bias.weight_loader = self.weight_loader
- else:
- self.register_parameter("bias", None)
def forward(self, x: torch.Tensor) -> torch.Tensor:
raise NotImplementedError
- def _shard_weight(self, loaded_weight: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:
- return loaded_weight
-
- def _shard_weight_scale(self, loaded_scale: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:
- return loaded_scale
-
- def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id=None):
- param.data.copy_(self._shard_weight(loaded_weight, loaded_shard_id))
-
- def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):
- device = self.weight.device
- shard = self._shard_weight(loaded_weight, loaded_shard_id)
- if self.qweight_t.numel() == 0:
- self.qweight_t = torch.empty(
- (self.input_size, self.output_size),
- dtype=torch.int8,
- device=device,
- )
- if shard.dtype != torch.int8:
- shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)
- else:
- shard = shard.to(device=device, non_blocking=True)
- # Store directly in transposed layout to avoid an extra full-size transpose pass
- # after loading.
- self.qweight_t.t().copy_(shard)
- shard_key = loaded_shard_id if loaded_shard_id is not None else 0
- self._quant_data_loaded.add(shard_key)
-
- def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):
- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device
- shard_scale = self._shard_weight_scale(loaded_scale, loaded_shard_id).contiguous()
- if shard_scale.dim() == 2 and shard_scale.size(1) == 1:
- shard_scale = shard_scale.squeeze(1)
- self.qweight_scale = shard_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)
- shard_key = loaded_shard_id if loaded_shard_id is not None else 0
- self._quant_scale_loaded.add(shard_key)
-
- def quant_input_scale_loader(self, loaded_scale: torch.Tensor):
- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device
- self.input_scale = loaded_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)
-
- def quant_output_scale_loader(self, loaded_scale: torch.Tensor):
- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device
- self.output_scale = loaded_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)
-
- def finalize_quantized(self):
- if self.qweight_scale.numel() == 0:
- return
- if len(self._quant_data_loaded) < self._quant_expected_shards:
- return
- if len(self._quant_scale_loaded) < self._quant_expected_shards:
- return
- if self.qweight_data.numel() == 0 and self.qweight_t.numel() == 0:
- return
- if self.qweight_t.numel() == 0:
- # Fallback path for loaders that still filled qweight_data (older checkpoints
- # or custom loaders). Keep memory usage low by releasing qweight_data immediately
- # after conversion.
- self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()
- self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)
- self.qweight_scale_fp32 = self.qweight_scale.to(dtype=torch.float32)
- if not torch.is_tensor(self.input_scale):
- raise RuntimeError(f"Invalid input_scale type: {type(self.input_scale)}")
- input_scale_flat = self.input_scale.reshape(-1)
- if input_scale_flat.numel() != 1:
- raise RuntimeError(
- f"Expected scalar input_scale, got shape={tuple(self.input_scale.shape)}"
- )
- input_scale_value = float(input_scale_flat[0].item())
- if not math.isfinite(input_scale_value) or input_scale_value <= 0:
- raise RuntimeError(f"Invalid input_scale value: {input_scale_value}")
- self._input_scale_value = max(input_scale_value, 1e-8)
- self.use_triton_int8 = bool(_TRITON_AVAILABLE and self.qweight_t.is_cuda)
- self.use_int8_weight = True
- device = self.qweight_t.device
- if self.weight.numel() != 0:
- self.weight = nn.Parameter(torch.empty(0, device=device, dtype=torch.bfloat16), requires_grad=False)
- if self.bias is not None:
- self.bias = nn.Parameter(self.bias.data.to(device=device), requires_grad=False)
-
- def _quant_int8_mm(self, x: torch.Tensor) -> torch.Tensor:
- x_shape = x.shape
- x2d = x.reshape(-1, x_shape[-1])
- if self.use_triton_int8 and x2d.is_cuda and self.qweight_t.numel() != 0 and self.qweight_scale_fp32.numel() != 0:
- y = _run_triton_fused_int8_mm(
- x2d,
- self.qweight_t,
- self.qweight_scale_fp32,
- )
- return y.view(*x_shape[:-1], y.size(-1))
-
- # Fallback path: use Quanto qbytes_mm for dynamic activation quantization parity.
- if not hasattr(torch.ops, "quanto") or not hasattr(torch.ops.quanto, "qbytes_mm"):
- raise RuntimeError("quanto.qbytes_mm op unavailable for int8 fallback path")
- if self.qweight_t.numel() == 0 and self.qweight_data.numel() != 0:
- self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()
- self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)
- qweight = self.qweight_t.transpose(0, 1).contiguous()
- scales = self.qweight_scale_fp32
- if scales.numel() == 0:
- scales = self.qweight_scale.to(torch.float32)
- scales = _flatten_scale(scales).reshape(-1, 1).contiguous()
- out = torch.ops.quanto.qbytes_mm(x2d, qweight, scales).to(x2d.dtype)
- return out.view(*x_shape[:-1], out.size(-1))
-
- def prepare_for_quantized_load(self):
- device = None
- if self.qweight_t.numel() != 0:
- device = self.qweight_t.device
- elif self.qweight_data.numel() != 0:
- device = self.qweight_data.device
- elif self.weight.numel() != 0:
- device = self.weight.device
- elif self.qweight_scale.numel() != 0:
- device = self.qweight_scale.device
- else:
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- if getattr(device, "type", None) == "meta":
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
- if self.weight.numel() != 0:
- self.weight = nn.Parameter(torch.empty(0, device=device, dtype=torch.bfloat16), requires_grad=False)
-
- # Reset all quantized buffers/state so each reload is complete and deterministic.
- self.qweight_data = torch.empty(0, dtype=torch.int8, device=device)
- self.qweight_t = torch.empty(0, dtype=torch.int8, device=device)
- self.qweight_scale = torch.empty(0, dtype=torch.bfloat16, device=device)
- self.qweight_scale_fp32 = torch.empty(0, dtype=torch.float32, device=device)
- self.input_scale = torch.ones((), dtype=torch.bfloat16, device=device)
- self.output_scale = torch.ones((), dtype=torch.bfloat16, device=device)
- self.use_int8_weight = False
- self.use_triton_int8 = False
- self._input_scale_value = 1.0
- self._quant_data_loaded.clear()
- self._quant_scale_loaded.clear()
-
-
-class ReplicatedLinear(LinearBase):
-
- def __init__(
- self,
- input_size: int,
- output_size: int,
- bias: bool = False,
- ):
- super().__init__(input_size, output_size, bias)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- if self.use_int8_weight:
- y = self._quant_int8_mm(x)
- if self.bias is not None:
- y = y + self.bias
- return y
- return F.linear(x, self.weight, self.bias)
-
-
class ColumnParallelLinear(LinearBase):
def __init__(
@@ -271,144 +33,12 @@ class ColumnParallelLinear(LinearBase):
bias: bool = False,
):
tp_size = _get_tp_info()[1]
- super().__init__(input_size, divide(output_size, tp_size), bias, 0)
-
- def _shard_weight(self, loaded_weight: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:
- shard_size = divide(loaded_weight.size(self.tp_dim), self.tp_size)
- start_idx = self.tp_rank * shard_size
- return loaded_weight.narrow(self.tp_dim, start_idx, shard_size)
-
- def _shard_weight_scale(self, loaded_scale: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:
- if loaded_scale.dim() == 0:
- return loaded_scale
- shard_size = divide(loaded_scale.size(0), self.tp_size)
- start_idx = self.tp_rank * shard_size
- return loaded_scale.narrow(0, start_idx, shard_size)
+ super().__init__(input_size, divide(output_size, tp_size), bias)
def forward(self, x: torch.Tensor) -> torch.Tensor:
- if self.use_int8_weight:
- y = self._quant_int8_mm(x)
- if self.bias is not None:
- y = y + self.bias
- return y
return F.linear(x, self.weight, self.bias)
-class MergedColumnParallelLinear(ColumnParallelLinear):
-
- def __init__(
- self,
- input_size: int,
- output_sizes: list[int],
- bias: bool = False,
- ):
- self.output_sizes = output_sizes
- super().__init__(input_size, sum(output_sizes), bias)
- self._quant_expected_shards = len(output_sizes)
-
- def _merged_shard_meta(self, loaded_shard_id: int):
- shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size
- shard_size = self.output_sizes[loaded_shard_id] // self.tp_size
- return shard_offset, shard_size
-
- def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int):
- param_data = param.data
- shard_offset, shard_size = self._merged_shard_meta(loaded_shard_id)
- param_slice = param_data.narrow(self.tp_dim, shard_offset, shard_size)
- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]
- param_slice.copy_(shard)
-
- def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):
- device = self.weight.device
- if self.qweight_t.numel() == 0:
- self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)
- shard_offset, shard_size = self._merged_shard_meta(int(loaded_shard_id))
- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]
- if shard.dtype != torch.int8:
- shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)
- else:
- shard = shard.to(device=device, non_blocking=True)
- self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())
- self._quant_data_loaded.add(int(loaded_shard_id))
-
- def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):
- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device
- if self.qweight_scale.numel() == 0:
- self.qweight_scale = torch.empty((self.output_size,), dtype=torch.bfloat16, device=device)
- shard_offset, shard_size = self._merged_shard_meta(int(loaded_shard_id))
- scale_slice = self.qweight_scale.narrow(0, shard_offset, shard_size)
- shard = loaded_scale.chunk(self.tp_size, 0)[self.tp_rank]
- if shard.dim() == 2 and shard.size(1) == 1:
- shard = shard.squeeze(1)
- scale_slice.copy_(shard.to(device=device, dtype=torch.bfloat16, non_blocking=True))
- self._quant_scale_loaded.add(int(loaded_shard_id))
-
-
-class QKVParallelLinear(ColumnParallelLinear):
-
- def __init__(
- self,
- hidden_size: int,
- head_size: int,
- total_num_heads: int,
- total_num_kv_heads: int | None = None,
- bias: bool = False,
- ):
- tp_size = _get_tp_info()[1]
- total_num_kv_heads = total_num_kv_heads or total_num_heads
- self.head_size = head_size
- self.num_heads = divide(total_num_heads, tp_size)
- self.num_kv_heads = divide(total_num_kv_heads, tp_size)
- output_size = (total_num_heads + 2 * total_num_kv_heads) * self.head_size
- super().__init__(hidden_size, output_size, bias)
- self._quant_expected_shards = 3
-
- def _qkv_offset_size(self, loaded_shard_id: str):
- assert loaded_shard_id in ["q", "k", "v"]
- if loaded_shard_id == "q":
- shard_size = self.num_heads * self.head_size
- shard_offset = 0
- elif loaded_shard_id == "k":
- shard_size = self.num_kv_heads * self.head_size
- shard_offset = self.num_heads * self.head_size
- else:
- shard_size = self.num_kv_heads * self.head_size
- shard_offset = self.num_heads * self.head_size + self.num_kv_heads * self.head_size
- return shard_offset, shard_size
-
- def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str):
- param_data = param.data
- shard_offset, shard_size = self._qkv_offset_size(loaded_shard_id)
- param_slice = param_data.narrow(self.tp_dim, shard_offset, shard_size)
- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]
- param_slice.copy_(shard)
-
- def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):
- device = self.weight.device
- if self.qweight_t.numel() == 0:
- self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)
- shard_offset, shard_size = self._qkv_offset_size(str(loaded_shard_id))
- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]
- if shard.dtype != torch.int8:
- shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)
- else:
- shard = shard.to(device=device, non_blocking=True)
- self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())
- self._quant_data_loaded.add(str(loaded_shard_id))
-
- def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):
- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device
- if self.qweight_scale.numel() == 0:
- self.qweight_scale = torch.empty((self.output_size,), dtype=torch.bfloat16, device=device)
- shard_offset, shard_size = self._qkv_offset_size(str(loaded_shard_id))
- scale_slice = self.qweight_scale.narrow(0, shard_offset, shard_size)
- shard = loaded_scale.chunk(self.tp_size, 0)[self.tp_rank]
- if shard.dim() == 2 and shard.size(1) == 1:
- shard = shard.squeeze(1)
- scale_slice.copy_(shard.to(device=device, dtype=torch.bfloat16, non_blocking=True))
- self._quant_scale_loaded.add(str(loaded_shard_id))
-
-
class RowParallelLinear(LinearBase):
def __init__(
@@ -418,24 +48,10 @@ class RowParallelLinear(LinearBase):
bias: bool = False,
):
tp_size = _get_tp_info()[1]
- super().__init__(divide(input_size, tp_size), output_size, bias, 1)
-
- def _shard_weight(self, loaded_weight: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:
- shard_size = divide(loaded_weight.size(self.tp_dim), self.tp_size)
- start_idx = self.tp_rank * shard_size
- return loaded_weight.narrow(self.tp_dim, start_idx, shard_size)
-
- def _shard_weight_scale(self, loaded_scale: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:
- # Output rows are not sharded in RowParallelLinear.
- return loaded_scale
+ super().__init__(divide(input_size, tp_size), output_size, bias)
def forward(self, x: torch.Tensor) -> torch.Tensor:
- if self.use_int8_weight:
- y = self._quant_int8_mm(x)
- if self.bias is not None and self.tp_rank == 0:
- y = y + self.bias
- else:
- y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None)
+ y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None)
if self.tp_size > 1:
dist.all_reduce(y)
return y
diff --git a/shared/llm_engines/nanovllm/layers/rotary_embedding.py b/shared/llm_engines/nanovllm/layers/rotary_embedding.py
index acaeffd..3227623 100644
--- a/shared/llm_engines/nanovllm/layers/rotary_embedding.py
+++ b/shared/llm_engines/nanovllm/layers/rotary_embedding.py
@@ -13,6 +13,16 @@ def apply_rotary_emb(
y2 = x2 * cos + x1 * sin
return torch.cat((y1, y2), dim=-1).to(x.dtype)
+@lru_cache(1)
+def _build_cache(device: torch.device | str | None, base, rotary_dim, max_position_embeddings ) -> torch.Tensor:
+ inv_freq = 1.0 / (
+ base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device) / rotary_dim)
+ )
+ t = torch.arange(max_position_embeddings, dtype=torch.float32, device=device)
+ freqs = torch.einsum("i,j -> ij", t, inv_freq)
+ cos = freqs.cos()
+ sin = freqs.sin()
+ return torch.cat((cos, sin), dim=-1).unsqueeze_(1)
class RotaryEmbedding(nn.Module):
@@ -27,28 +37,12 @@ class RotaryEmbedding(nn.Module):
self.head_size = head_size
self.rotary_dim = rotary_dim
self.max_position_embeddings = max_position_embeddings
- self.base = float(base)
+ base = float(base)
+ self.base = base
assert rotary_dim == head_size
- cache = self._build_cache(device=None)
+ cache = _build_cache("cpu", base, rotary_dim, max_position_embeddings )
self.register_buffer("cos_sin_cache", cache, persistent=False)
- def _build_cache(self, device: torch.device | str | None) -> torch.Tensor:
- inv_freq = 1.0 / (
- self.base ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32, device=device) / self.rotary_dim)
- )
- t = torch.arange(self.max_position_embeddings, dtype=torch.float32, device=device)
- freqs = torch.einsum("i,j -> ij", t, inv_freq)
- cos = freqs.cos()
- sin = freqs.sin()
- return torch.cat((cos, sin), dim=-1).unsqueeze_(1)
-
- def materialize_cache(self, device: torch.device | str | None = None):
- target_device = device
- if target_device is None:
- target_device = self.cos_sin_cache.device
- if getattr(target_device, "type", None) == "meta":
- target_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- self.cos_sin_cache = self._build_cache(target_device)
@torch.compile
def forward(
@@ -63,15 +57,3 @@ class RotaryEmbedding(nn.Module):
key = apply_rotary_emb(key, cos, sin)
return query, key
-
-@lru_cache(1)
-def get_rope(
- head_size: int,
- rotary_dim: int,
- max_position: int,
- base: float,
- rope_scaling: dict | None = None,
-):
- assert rope_scaling is None
- rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base)
- return rotary_emb
diff --git a/shared/llm_engines/nanovllm/llm.py b/shared/llm_engines/nanovllm/llm.py
index 4f51a44..0266054 100644
--- a/shared/llm_engines/nanovllm/llm.py
+++ b/shared/llm_engines/nanovllm/llm.py
@@ -1,4 +1,4 @@
-from nanovllm.engine.llm_engine import LLMEngine
+from .engine.llm_engine import LLMEngine
class LLM(LLMEngine):
diff --git a/shared/llm_engines/nanovllm/models/__init__.py b/shared/llm_engines/nanovllm/models/__init__.py
index 692e251..67d5754 100644
--- a/shared/llm_engines/nanovllm/models/__init__.py
+++ b/shared/llm_engines/nanovllm/models/__init__.py
@@ -1,52 +1,3 @@
-import importlib
-from typing import Iterable
-from torch import nn
+from .qwen3 import Qwen3ForCausalLM
-
-_ARCH_REGISTRY = {
- "Qwen3ForCausalLM": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),
-}
-
-_MODEL_TYPE_REGISTRY = {
- "qwen3": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),
-}
-
-
-def _load_model_cls(module_name: str, class_name: str):
- module = importlib.import_module(module_name)
- model_cls = getattr(module, class_name, None)
- if model_cls is None:
- raise RuntimeError(f"Model class '{class_name}' was not found in '{module_name}'.")
- if not isinstance(model_cls, type) or not issubclass(model_cls, nn.Module):
- raise RuntimeError(f"Resolved '{module_name}.{class_name}' is not a torch.nn.Module subclass.")
- return model_cls
-
-
-def _iter_architectures(hf_config) -> Iterable[str]:
- architectures = getattr(hf_config, "architectures", None)
- if isinstance(architectures, (list, tuple)):
- for architecture in architectures:
- if isinstance(architecture, str) and len(architecture) > 0:
- yield architecture
-
-
-def resolve_model_class(hf_config):
- for architecture in _iter_architectures(hf_config):
- entry = _ARCH_REGISTRY.get(architecture, None)
- if entry is not None:
- return _load_model_cls(entry[0], entry[1])
-
- model_type = str(getattr(hf_config, "model_type", "")).strip().lower()
- if model_type:
- entry = _MODEL_TYPE_REGISTRY.get(model_type, None)
- if entry is not None:
- return _load_model_cls(entry[0], entry[1])
-
- supported_architectures = ", ".join(sorted(_ARCH_REGISTRY.keys()))
- supported_model_types = ", ".join(sorted(_MODEL_TYPE_REGISTRY.keys()))
- found_architectures = ", ".join(list(_iter_architectures(hf_config))) or "<none>"
- raise RuntimeError(
- "nanovllm does not support this model architecture. "
- f"Found architectures: {found_architectures}; model_type: '{model_type or '<none>'}'. "
- f"Supported architectures: {supported_architectures}; supported model_type values: {supported_model_types}."
- )
+__all__ = ["Qwen3ForCausalLM"]
diff --git a/shared/llm_engines/nanovllm/models/qwen3.py b/shared/llm_engines/nanovllm/models/qwen3.py
index 7b98783..550a358 100644
--- a/shared/llm_engines/nanovllm/models/qwen3.py
+++ b/shared/llm_engines/nanovllm/models/qwen3.py
@@ -3,12 +3,12 @@ from torch import nn
import torch.distributed as dist
from transformers import Qwen3Config
-from nanovllm.layers.activation import SiluAndMul
-from nanovllm.layers.attention import Attention
-from nanovllm.layers.layernorm import RMSNorm
-from nanovllm.layers.linear import QKVParallelLinear, MergedColumnParallelLinear, RowParallelLinear
-from nanovllm.layers.rotary_embedding import get_rope
-from nanovllm.layers.embed_head import VocabParallelEmbedding, ParallelLMHead
+from ..layers.activation import SiluAndMul
+from ..layers.attention import Attention
+from ..layers.layernorm import RMSNorm
+from ..layers.linear import ColumnParallelLinear, RowParallelLinear
+from ..layers.rotary_embedding import RotaryEmbedding
+from ..layers.embed_head import VocabParallelEmbedding, ParallelLMHead
def _get_tp_size():
@@ -45,31 +45,18 @@ class Qwen3Attention(nn.Module):
self.scaling = self.head_dim ** -0.5
self.qkv_bias = qkv_bias
- self.qkv_proj = QKVParallelLinear(
- hidden_size,
- self.head_dim,
- self.total_num_heads,
- self.total_num_kv_heads,
- bias=qkv_bias,
- )
+ self.q_proj = ColumnParallelLinear(hidden_size, self.total_num_heads * self.head_dim, bias=qkv_bias)
+ self.k_proj = ColumnParallelLinear(hidden_size, self.total_num_kv_heads * self.head_dim, bias=qkv_bias)
+ self.v_proj = ColumnParallelLinear(hidden_size, self.total_num_kv_heads * self.head_dim, bias=qkv_bias)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
hidden_size,
bias=False,
)
- self.rotary_emb = get_rope(
- self.head_dim,
- rotary_dim=self.head_dim,
- max_position=max_position,
- base=rope_theta,
- rope_scaling=rope_scaling,
- )
- self.attn = Attention(
- self.num_heads,
- self.head_dim,
- self.scaling,
- self.num_kv_heads,
- )
+ self.rotary_emb = RotaryEmbedding(self.head_dim, self.head_dim, max_position, rope_theta)
+
+ self.attn = Attention( self.num_heads, self.head_dim, self.scaling, self.num_kv_heads)
+
if not self.qkv_bias:
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
@@ -79,8 +66,9 @@ class Qwen3Attention(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
- qkv = self.qkv_proj(hidden_states)
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+ q = self.q_proj(hidden_states)
+ k = self.k_proj(hidden_states)
+ v = self.v_proj(hidden_states)
q = q.view(-1, self.num_heads, self.head_dim)
k = k.view(-1, self.num_kv_heads, self.head_dim)
v = v.view(-1, self.num_kv_heads, self.head_dim)
@@ -102,11 +90,8 @@ class Qwen3MLP(nn.Module):
hidden_act: str,
) -> None:
super().__init__()
- self.gate_up_proj = MergedColumnParallelLinear(
- hidden_size,
- [intermediate_size] * 2,
- bias=False,
- )
+ self.gate_proj = ColumnParallelLinear(hidden_size, intermediate_size, bias=False)
+ self.up_proj = ColumnParallelLinear(hidden_size, intermediate_size, bias=False)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
@@ -116,8 +101,9 @@ class Qwen3MLP(nn.Module):
self.act_fn = SiluAndMul()
def forward(self, x):
- gate_up = self.gate_up_proj(x)
- x = self.act_fn(gate_up)
+ gate = self.gate_proj(x)
+ up = self.up_proj(x)
+ x = self.act_fn(torch.cat((gate, up), dim=-1))
x = self.down_proj(x)
return x
@@ -189,14 +175,6 @@ class Qwen3Model(nn.Module):
class Qwen3ForCausalLM(nn.Module):
- packed_modules_mapping = {
- "q_proj": ("qkv_proj", "q"),
- "k_proj": ("qkv_proj", "k"),
- "v_proj": ("qkv_proj", "v"),
- "gate_proj": ("gate_up_proj", 0),
- "up_proj": ("gate_up_proj", 1),
- }
-
def __init__(
self,
config: Qwen3Config
@@ -205,8 +183,7 @@ class Qwen3ForCausalLM(nn.Module):
self.model = Qwen3Model(config)
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
- # Proxy attributes for weight loading compatibility
- # Some model weights use "embed_tokens" instead of "model.embed_tokens"
+ # Proxy attributes to keep the top-level module layout simple for runtime helpers.
@property
def embed_tokens(self):
return self.model.embed_tokens
diff --git a/shared/llm_engines/nanovllm/utils/loader.py b/shared/llm_engines/nanovllm/utils/loader.py
deleted file mode 100644
index 0a5b490..0000000
--- a/shared/llm_engines/nanovllm/utils/loader.py
+++ /dev/null
@@ -1,297 +0,0 @@
-import gc
-import os
-from collections import OrderedDict
-from glob import glob
-import torch
-from torch import nn
-from safetensors import safe_open
-
-
-def default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor):
- if param.is_meta:
- raise RuntimeError("default_weight_loader cannot copy into meta parameter directly.")
- if loaded_weight.device != param.device or loaded_weight.dtype != param.dtype:
- loaded_weight = loaded_weight.to(
- device=param.device,
- dtype=param.dtype,
- non_blocking=True,
- )
- param.data.copy_(loaded_weight)
-
-
-def _coerce_loaded_weight_to_param(param: nn.Parameter, loaded_weight: torch.Tensor) -> torch.Tensor:
- target_device = param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")
- if loaded_weight.device != target_device:
- loaded_weight = loaded_weight.to(device=target_device, non_blocking=True)
- if loaded_weight.dtype != param.dtype:
- loaded_weight = loaded_weight.to(dtype=param.dtype, non_blocking=True)
- return loaded_weight
-
-
-def _restore_param_attributes(new_param: nn.Parameter, src_param: nn.Parameter) -> None:
- for attr_name, attr_value in src_param.__dict__.items():
- setattr(new_param, attr_name, attr_value)
-
-
-def _replace_meta_parameter(
- parent: nn.Module,
- param_name: str,
- loaded_weight: torch.Tensor,
- src_param: nn.Parameter,
-) -> nn.Parameter:
- new_param = nn.Parameter(
- loaded_weight,
- requires_grad=src_param.requires_grad,
- )
- parent._parameters[param_name] = new_param
- _restore_param_attributes(new_param, src_param)
- return new_param
-
-
-def _replace_tied_meta_parameter(
- model: nn.Module,
- old_param: nn.Parameter,
- loaded_weight: torch.Tensor,
-) -> nn.Parameter:
- new_param = nn.Parameter(
- loaded_weight,
- requires_grad=old_param.requires_grad,
- )
- _restore_param_attributes(new_param, old_param)
- for module in model.modules():
- for name, param in list(module._parameters.items()):
- if param is old_param:
- module._parameters[name] = new_param
- return new_param
-
-
-def _should_replace_param_globally(model: nn.Module, param: nn.Parameter) -> bool:
- tied_ids = getattr(model, "_vllm_tied_param_ids", None)
- return bool(tied_ids) and id(param) in tied_ids
-
-
-def _candidate_weight_names(weight_name: str):
- candidates = [weight_name]
- if weight_name.startswith("model."):
- candidates.append(weight_name[6:])
- else:
- candidates.append(f"model.{weight_name}")
- return candidates
-
-
-def _get_parameter_target(model: nn.Module, weight_name: str):
- for candidate in _candidate_weight_names(weight_name):
- if "." in candidate:
- parent_name, param_name = candidate.rsplit(".", 1)
- else:
- parent_name, param_name = "", candidate
- try:
- parent = model if parent_name == "" else model.get_submodule(parent_name)
- except Exception:
- continue
- param = parent._parameters.get(param_name)
- if param is not None and isinstance(param, nn.Parameter):
- return parent, param_name, param
- return None
-
-
-def _get_submodule_safe(model: nn.Module, module_name: str):
- for candidate in _candidate_weight_names(module_name):
- if candidate == "":
- return model
- try:
- return model.get_submodule(candidate)
- except Exception:
- pass
- return None
-
-
-def _list_safetensor_files(path: str) -> list[str]:
- if os.path.isfile(path) and path.endswith(".safetensors"):
- return [path]
- return glob(os.path.join(path, "*.safetensors"))
-
-
-class WeightStore:
- def __init__(self, path: str, mode: str = "lazy"):
- self.path = path
- self.mode = (mode or "lazy").lower()
- self.files = _list_safetensor_files(path)
- if not self.files:
- raise FileNotFoundError(f"No .safetensors files found in {path}")
- self._weight_to_file = {}
- self._file_to_weight_names = OrderedDict()
- self._pinned_weights = {}
- self.is_quanto_int8 = False
- for file in self.files:
- with safe_open(file, "pt", "cpu", writable_tensors=False) as f:
- for key in f.keys():
- self._weight_to_file[key] = file
- self._file_to_weight_names.setdefault(file, []).append(key)
- if key.endswith(".weight._data"):
- self.is_quanto_int8 = True
- if self.mode == "pinned":
- self._preload_pinned()
-
- def _preload_pinned(self):
- for file, keys in self._file_to_weight_names.items():
- with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:
- for key in keys:
- tensor = handle.get_tensor(key)
- if tensor.device.type != "cpu":
- tensor = tensor.cpu()
- if not tensor.is_pinned():
- tensor = tensor.pin_memory()
- self._pinned_weights[key] = tensor
-
- def close(self):
- self._weight_to_file.clear()
- self._file_to_weight_names.clear()
- self._pinned_weights.clear()
-
- def __del__(self):
- try:
- self.close()
- except Exception:
- pass
-
- def get_tensor(self, key: str) -> torch.Tensor:
- if self.mode == "pinned":
- return self._pinned_weights[key]
- file = self._weight_to_file.get(key)
- if file is None:
- raise KeyError(key)
- with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:
- return handle.get_tensor(key)
-
-
-def load_model(
- model: nn.Module,
- path: str,
- weight_store: WeightStore | None = None,
- clone_loaded_tensors: bool = False,
-):
- packed_modules_mapping = getattr(model, "packed_modules_mapping", {})
- if weight_store is None:
- safetensor_files = _list_safetensor_files(path)
- if not safetensor_files:
- raise FileNotFoundError(f"No .safetensors files found in {path}")
- for file in safetensor_files:
- with safe_open(file, "pt", "cpu", writable_tensors=False) as f:
- for weight_name in f.keys():
- tensor = f.get_tensor(weight_name)
- _apply_weight(model, packed_modules_mapping, weight_name, tensor)
- _finalize_quantized_modules(model)
- return
- for file in weight_store.files:
- file_weights = weight_store._file_to_weight_names.get(file, [])
- if not file_weights:
- continue
- with safe_open(file, "pt", "cuda", writable_tensors = False, streaming = True) as f:
- file_weights = f.reorder_for_streaming(file_weights)
- for weight_name in file_weights:
- tensor = f.get_tensor(weight_name)
- if clone_loaded_tensors:
- tensor = tensor.clone()
- try:
- _apply_weight(model, packed_modules_mapping, weight_name, tensor)
- finally:
- del tensor
- f = None
- if torch.cuda.is_available():
- torch.cuda.synchronize()
- gc.collect()
- _finalize_quantized_modules(model)
-
-
-def _apply_weight(model: nn.Module, packed_modules_mapping, weight_name: str, tensor: torch.Tensor):
- quant_suffix = None
- if weight_name.endswith(".weight._data"):
- quant_suffix = "qdata"
- elif weight_name.endswith(".weight._scale"):
- quant_suffix = "qscale"
- elif weight_name.endswith(".input_scale"):
- quant_suffix = "input_scale"
- elif weight_name.endswith(".output_scale"):
- quant_suffix = "output_scale"
-
- mapped_name = weight_name
- shard_id = None
- for src_name, mapped in packed_modules_mapping.items():
- if src_name in weight_name:
- mapped_target = mapped[0] if isinstance(mapped, tuple) else mapped
- mapped_name = weight_name.replace(src_name, mapped_target)
- shard_id = mapped[1] if isinstance(mapped, tuple) and len(mapped) > 1 else None
- break
-
- if quant_suffix is not None:
- if quant_suffix == "qdata":
- module_name = mapped_name[:-len(".weight._data")]
- loader_name = "quant_weight_data_loader"
- elif quant_suffix == "qscale":
- module_name = mapped_name[:-len(".weight._scale")]
- loader_name = "quant_weight_scale_loader"
- elif quant_suffix == "input_scale":
- module_name = mapped_name[:-len(".input_scale")]
- loader_name = "quant_input_scale_loader"
- else:
- module_name = mapped_name[:-len(".output_scale")]
- loader_name = "quant_output_scale_loader"
- module = _get_submodule_safe(model, module_name)
- if module is None:
- print(f"[loader] Warning: Module not found: {module_name}")
- return
- loader_fn = getattr(module, loader_name, None)
- if loader_fn is None:
- print(f"[loader] Warning: Quant loader not found on module: {module_name}")
- return
- if shard_id is not None and quant_suffix in ("qdata", "qscale"):
- loader_fn(tensor, shard_id)
- else:
- loader_fn(tensor)
- return
-
- target = _get_parameter_target(model, mapped_name)
- if target is None:
- print(f"[loader] Warning: Parameter not found: {mapped_name}")
- return
- parent, param_name, param = target
- weight_loader = getattr(param, "weight_loader", None)
-
- if shard_id is not None and weight_loader is not None:
- if param.is_meta:
- replacement_device = (
- param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")
- )
- replacement = torch.empty(param.shape, device=replacement_device, dtype=param.dtype)
- if _should_replace_param_globally(model, param):
- param = _replace_tied_meta_parameter(model, param, replacement)
- else:
- param = _replace_meta_parameter(parent, param_name, replacement, param)
- try:
- weight_loader(param, tensor, shard_id)
- except TypeError:
- weight_loader(param, tensor)
- return
-
- if param.is_meta:
- loaded_weight = _coerce_loaded_weight_to_param(param, tensor)
- if loaded_weight.shape != param.shape:
- raise RuntimeError(
- f"Weight shape mismatch for '{mapped_name}': expected {tuple(param.shape)}, got {tuple(loaded_weight.shape)}"
- )
- if _should_replace_param_globally(model, param):
- _replace_tied_meta_parameter(model, param, loaded_weight)
- else:
- _replace_meta_parameter(parent, param_name, loaded_weight, param)
- return
- if weight_loader is None:
- weight_loader = default_weight_loader
- weight_loader(param, tensor)
-
-
-def _finalize_quantized_modules(model: nn.Module):
- for module in model.modules():
- finalize = getattr(module, "finalize_quantized", None)
- if callable(finalize):
- finalize()
diff --git a/shared/llm_engines/nanovllm/vllm_support.py b/shared/llm_engines/nanovllm/vllm_support.py
index f8c195e..7434544 100644
--- a/shared/llm_engines/nanovllm/vllm_support.py
+++ b/shared/llm_engines/nanovllm/vllm_support.py
@@ -1,3 +1,5 @@
+from typing import Any, Callable, Optional
+
_PROBE_CACHE = None
_WARNED_REQUESTED_VLLM_UNAVAILABLE = False
@@ -31,61 +33,6 @@ def _check_flash_attention_2():
return True, "ok"
-def _load_linear_module():
- import importlib.util
- import os
-
- base_dir = os.path.dirname(os.path.abspath(__file__))
- linear_path = os.path.join(base_dir, "layers", "linear.py")
- if not os.path.isfile(linear_path):
- raise RuntimeError(f"Missing nanovllm linear kernel file: {linear_path}")
-
- spec = importlib.util.spec_from_file_location("nanovllm_linear_probe", linear_path)
- if spec is None or spec.loader is None:
- raise RuntimeError("Unable to build import spec for nanovllm linear kernel probe")
- module = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(module)
- return module
-
-
-def _check_triton_int8_kernel():
- import inspect
- import torch
-
- if not torch.cuda.is_available():
- return False, "CUDA is not available"
-
- try:
- linear_module = _load_linear_module()
- if not getattr(linear_module, "_TRITON_AVAILABLE", False):
- return False, "nanovllm Triton path is disabled"
-
- run_kernel = getattr(linear_module, "_run_triton_fused_int8_mm", None)
- if run_kernel is None:
- return False, "nanovllm Triton int8 kernel entrypoint is missing"
-
- device = torch.device("cuda")
- x = torch.randn((2, 64), device=device, dtype=torch.bfloat16)
- qweight_t = torch.randint(-127, 128, (64, 32), device=device, dtype=torch.int8)
- qweight_scale = torch.ones((32,), device=device, dtype=torch.float32)
- # Support both legacy (4-arg) and current (3-arg) probe signatures.
- param_count = len(inspect.signature(run_kernel).parameters)
- if param_count >= 4:
- out = run_kernel(x, qweight_t, qweight_scale, 0.01)
- else:
- out = run_kernel(x, qweight_t, qweight_scale)
- torch.cuda.synchronize()
-
- if tuple(out.shape) != (2, 32):
- return False, f"Unexpected kernel output shape: {tuple(out.shape)}"
- if not torch.isfinite(out).all().item():
- return False, "Kernel output contains non-finite values"
- except Exception as exc:
- return False, f"Triton int8 kernel smoke test failed: {exc}"
-
- return True, "ok"
-
-
def probe_vllm_runtime(force=False):
global _PROBE_CACHE
if _PROBE_CACHE is not None and not force:
@@ -99,10 +46,7 @@ def probe_vllm_runtime(force=False):
flash_ok, flash_msg = _check_flash_attention_2()
checks["flash_attention_2"] = {"ok": flash_ok, "message": flash_msg}
- kernel_ok, kernel_msg = _check_triton_int8_kernel()
- checks["triton_int8_kernel"] = {"ok": kernel_ok, "message": kernel_msg}
-
- supported = triton_ok and flash_ok and kernel_ok
+ supported = triton_ok and flash_ok
result = {
"supported": supported,
"preferred_engine": "vllm" if supported else "legacy",
@@ -114,6 +58,7 @@ def probe_vllm_runtime(force=False):
def resolve_lm_decoder_engine(requested_engine):
+ requested_engine = str(requested_engine or "").strip().lower()
probe_result = probe_vllm_runtime()
supported = bool(probe_result.get("supported", False))
if requested_engine == "vllm":
@@ -136,4 +81,264 @@ def resolve_lm_decoder_engine(requested_engine):
return "legacy"
if requested_engine == "":
return "vllm" if supported else "legacy"
- return requested_engine
+ if requested_engine == "legacy":
+ return "legacy"
+ print(f"[LM] Unknown decoder engine '{requested_engine}', falling back to 'legacy'.")
+ return "legacy"
+
+
+class NanoVllmTextEngine:
+ keep_loaded_for_phase2 = True
+
+ def __init__(self, model, model_path: str, tokenizer):
+ self.model = model
+ self.model_path = model_path
+ self.tokenizer = tokenizer
+ self._llm = None
+ self._sampling_params_cls = None
+ self._max_model_len_hint = None
+ self._max_num_seqs_hint = None
+ self._max_num_batched_tokens_hint = None
+ self._last_failure_reason = ""
+
+ @staticmethod
+ def _compute_runtime_hints(prompt_len: int, max_tokens: int, cfg_scale: float):
+ max_model_len = max(8, int(prompt_len) + int(max_tokens))
+ max_num_seqs = 2 if cfg_scale and cfg_scale > 1.0 else 1
+ max_num_batched_tokens = max_model_len * max_num_seqs
+ return max_model_len, max_num_seqs, max_num_batched_tokens
+
+ def _ensure_runtime_capacity(self, max_model_len: int, max_num_seqs: int, max_num_batched_tokens: int):
+ if self._max_model_len_hint is None:
+ self._max_model_len_hint = max_model_len
+ self._max_num_seqs_hint = max_num_seqs
+ self._max_num_batched_tokens_hint = max_num_batched_tokens
+ return
+
+ need_grow = (
+ max_model_len > int(self._max_model_len_hint)
+ or max_num_seqs > int(self._max_num_seqs_hint)
+ or max_num_batched_tokens > int(self._max_num_batched_tokens_hint)
+ )
+ if not need_grow:
+ return
+
+ self._max_model_len_hint = max_model_len
+ self._max_num_seqs_hint = max_num_seqs
+ self._max_num_batched_tokens_hint = max_num_batched_tokens
+ self.close()
+
+ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):
+ req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(
+ prompt_len=prompt_len,
+ max_tokens=max_tokens,
+ cfg_scale=cfg_scale,
+ )
+ self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)
+
+ def _ensure_llm(self):
+ if self._llm is not None:
+ return
+ try:
+ import torch._inductor.config as inductor_config
+
+ if bool(getattr(inductor_config, "split_reductions", False)):
+ inductor_config.split_reductions = False
+ except Exception:
+ pass
+ try:
+ from shared.llm_engines.nanovllm import LLM, SamplingParams
+ except Exception as exc: # pragma: no cover
+ raise RuntimeError(f"nano-vllm is not available for vllm engine: {exc}") from exc
+ if not self.model_path:
+ raise RuntimeError("vllm engine requires a model_path")
+ max_model_len = self._max_model_len_hint or 4096
+ max_num_seqs = self._max_num_seqs_hint or 1
+ max_num_batched_tokens = self._max_num_batched_tokens_hint or (max_model_len * max_num_seqs)
+ self._llm = LLM(
+ model=self.model_path,
+ enforce_eager=False,
+ tensor_parallel_size=1,
+ max_model_len=max_model_len,
+ max_num_seqs=max_num_seqs,
+ max_num_batched_tokens=max_num_batched_tokens,
+ tokenizer=self.tokenizer,
+ model_object=self.model,
+ )
+ self._sampling_params_cls = SamplingParams
+
+ def release_runtime_allocations(self):
+ if self._llm is None:
+ return
+ try:
+ self._llm.reset_runtime_state()
+ except Exception:
+ pass
+
+ def close(self):
+ llm = getattr(self, "_llm", None)
+ self._llm = None
+ if llm is not None:
+ try:
+ close_fn = getattr(llm, "close", None)
+ if callable(close_fn):
+ close_fn()
+ else:
+ try:
+ llm.reset_runtime_state()
+ except Exception:
+ pass
+ try:
+ llm.clear_graph_cache()
+ except Exception:
+ pass
+ exit_fn = getattr(llm, "exit", None)
+ if callable(exit_fn):
+ exit_fn()
+ except Exception:
+ pass
+ try:
+ del llm
+ except Exception:
+ pass
+ self._sampling_params_cls = None
+
+ def __del__(self):
+ self.close()
+
+ @staticmethod
+ def _extract_text_and_tokens(output_obj) -> tuple[str, list[int]]:
+ if output_obj is None:
+ return "", []
+ if isinstance(output_obj, dict):
+ text = str(output_obj.get("text", "") or "")
+ token_ids = output_obj.get("token_ids", []) or []
+ return text, [int(x) for x in token_ids]
+ if hasattr(output_obj, "outputs"):
+ outputs = getattr(output_obj, "outputs", None)
+ if outputs and len(outputs) > 0:
+ text = str(getattr(outputs[0], "text", "") or "")
+ token_ids = getattr(outputs[0], "token_ids", None)
+ if token_ids is None:
+ token_ids = getattr(outputs[0], "token_ids_list", []) or []
+ return text, [int(x) for x in token_ids] if token_ids else []
+ text = str(getattr(output_obj, "text", "") or "")
+ token_ids = getattr(output_obj, "token_ids", None)
+ if token_ids is None:
+ token_ids = []
+ return text, [int(x) for x in token_ids]
+
+ def get_last_failure_reason(self) -> str:
+ return self._last_failure_reason
+
+ def generate_text(
+ self,
+ prompt: str,
+ prompt_negative: str,
+ max_tokens: int,
+ temperature: Optional[float],
+ top_p: Optional[float],
+ top_k: Optional[int],
+ cfg_scale: float,
+ seed: Optional[int],
+ callback=None,
+ abort_fn: Optional[Callable[[], bool]] = None,
+ logits_processor: Optional[Any] = None,
+ logits_processor_update_state: Optional[Callable[[int], None]] = None,
+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,
+ progress_label: str = "LM text",
+ release_vram_after: bool = True,
+ ignore_eos: bool = False,
+ ):
+ del stop_checker
+ if abort_fn is not None and abort_fn():
+ if release_vram_after:
+ self.release_runtime_allocations()
+ return None
+ try:
+ prompt_len = len(self.tokenizer.encode(prompt))
+ except Exception:
+ prompt_len = 0
+ if cfg_scale > 1.0 and prompt_negative:
+ try:
+ prompt_len = max(prompt_len, len(self.tokenizer.encode(prompt_negative)))
+ except Exception:
+ pass
+
+ req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(
+ prompt_len=prompt_len,
+ max_tokens=max_tokens,
+ cfg_scale=cfg_scale,
+ )
+ self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)
+ self._ensure_llm()
+ if self._llm is None:
+ return None
+ try:
+ self._llm.reset()
+ except Exception:
+ pass
+
+ if callback is not None:
+ callback(
+ step_idx=-1,
+ override_num_inference_steps=max_tokens,
+ denoising_extra=f"{progress_label} 0/{max_tokens}",
+ progress_unit="tokens",
+ )
+
+ seed_value = None
+ if seed is not None:
+ try:
+ seed_value = int(seed)
+ except Exception:
+ seed_value = None
+ if seed_value is not None and seed_value < 0:
+ seed_value = None
+
+ temp = temperature if temperature is not None and temperature > 0 else 1e-5
+ sampling_params = self._sampling_params_cls(
+ temperature=temp,
+ max_tokens=max_tokens,
+ cfg_scale=max(cfg_scale, 1.0),
+ top_k=top_k if top_k is not None and top_k > 0 else None,
+ top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,
+ ignore_eos=bool(ignore_eos),
+ logits_processor=logits_processor,
+ logits_processor_update_state=logits_processor_update_state,
+ seed=seed_value,
+ )
+
+ text = ""
+ token_ids: list[int] = []
+ try:
+ outputs = self._llm.generate(
+ prompts=[prompt],
+ sampling_params=sampling_params,
+ use_tqdm=True,
+ unconditional_prompts=[prompt_negative] if cfg_scale > 1.0 else None,
+ )
+ if outputs:
+ text, token_ids = self._extract_text_and_tokens(outputs[0])
+ if (not text) and token_ids:
+ try:
+ text = self.tokenizer.decode(token_ids, skip_special_tokens=False)
+ except Exception:
+ text = ""
+ self._last_failure_reason = ""
+ except Exception as exc:
+ self._last_failure_reason = str(exc)
+ raise
+ finally:
+ if release_vram_after:
+ self.release_runtime_allocations()
+
+ if callback is not None:
+ callback(
+ step_idx=max(0, max_tokens - 1),
+ override_num_inference_steps=max_tokens,
+ denoising_extra=f"{progress_label} {max_tokens}/{max_tokens}",
+ progress_unit="tokens",
+ )
+
+ return {"token_ids": token_ids, "text": text}