Python
refactory vllm to use mmgp, ace step 1.5 cleanup
DeepBeepMeep10ef884e4e51
diff --git a/models/TTS/ace_step15/pipeline_ace_step15.py b/models/TTS/ace_step15/pipeline_ace_step15.pyindex 4bc9e2b..4e6a6b3 100644--- a/models/TTS/ace_step15/pipeline_ace_step15.py+++ b/models/TTS/ace_step15/pipeline_ace_step15.py@@ -7,20 +7,21 @@ import mathimport osimport randomimport re-import sysfrom typing import Anyimport torchimport torchaudioimport yamlfrom tqdm import tqdm-from transformers import AutoTokenizer, Qwen3ForCausalLM, Qwen3Model+from transformers import AutoTokenizer, Qwen3ForCausalLM as HfQwen3ForCausalLM, Qwen3Modelfrom mmgp import offloadfrom shared.utils.text_encoder_cache import TextEncoderCachefrom .models.autoencoder_oobleck import AutoencoderOobleckfrom .models.ace_step15_hf import AceStepConditionGenerationModel+from .qwen3_audio_codes import generate_audio_codes_with_engine_sampling+from .qwen3_lm_engines import Qwen3LegacyEngine, Qwen3LmEngine, Qwen3VllmEngine_DEFAULT_TIMBRE = [-1.3672e-01, -1.5820e-01, 5.8594e-01, -5.7422e-01, 3.0273e-02,@@ -111,8 +112,6 @@ class ACEStep15Pipeline:enable_lm: bool = True,ignore_lm_cache_seed: bool = False,lm_decoder_engine: str = "legacy",- lm_vllm_weight_mode: str = "lazy",- lm_vllm_free_mmap_after_cuda_copy: bool = True,device=None,dtype=torch.bfloat16,):@@ -137,26 +136,15 @@ class ACEStep15Pipeline:self.lm_tokenizer_dir = lm_tokenizer_dirself.silence_latent_path = silence_latent_pathself.ignore_lm_cache_seed = bool(ignore_lm_cache_seed)- self.lm_engine = (lm_decoder_engine or "legacy").strip().lower()- if self.lm_engine not in ("legacy", "pt", "vllm"):- self.lm_engine = "legacy"- self.lm_vllm_weight_mode = (lm_vllm_weight_mode or "lazy").strip().lower()- if self.lm_engine == "vllm" and self.lm_vllm_weight_mode not in ("lazy", "pinned"):- raise ValueError(- f"Unsupported Ace Step vllm LM weight mode '{self.lm_vllm_weight_mode}'. Supported: lazy, pinned."- )- self.lm_vllm_free_mmap_after_cuda_copy = bool(lm_vllm_free_mmap_after_cuda_copy)+ self.lm_engine = str(lm_decoder_engine or "legacy").strip().lower()+ if self.lm_engine not in {"legacy", "vllm"}:+ raise ValueError(f"Unsupported LM engine '{lm_decoder_engine}'. Expected one of: legacy, vllm.")self._interrupt = Falseself._early_stop = Falseself.loaded = Falseself._latent_hop_length = 1920- self.lm_code_cache = TextEncoderCache()- self._lm_engine_impl = None- self._lm_create_engine_fn = None- self._lm_legacy_generate_fn = None- self._lm_legacy_text_fn = Noneself._ref_metadata_processor = Noneself._ref_metadata_processor_class = Noneself._ref_caption_postprocess_fn = None@@ -165,12 +153,13 @@ class ACEStep15Pipeline:self._init_lm_hint_modules()self._load_tokenizers()self._load_text_encoder_2()- if self.enable_lm and self.lm_engine != "vllm":- self._load_lm()- else:- self.lm_model = Noneself._load_silence_latent()+ self.lm_code_cache = TextEncoderCache()+ self._lm_engine_impl: Qwen3LmEngine | None = None+ self._lm_last_failure_reason = ""+ self.lm_model = Noneif self.enable_lm:+ self._load_lm()self._init_lm_engine()self.loaded = True@@ -312,66 +301,45 @@ class ACEStep15Pipeline:def _load_lm(self):config_path = os.path.join(os.path.dirname(self.lm_weights_path), "config.json")+def _remap_lm_state_dict(state_dict, quantization_map=None, tied_weights_map=None):- # AceStep 5Hz LM weights are stored without a `model.` prefix.- if any(key.startswith("model.") for key in state_dict.keys()):- if "lm_head.weight" not in state_dict and "model.embed_tokens.weight" in state_dict:- state_dict["lm_head.weight"] = state_dict["model.embed_tokens.weight"]- return state_dict, quantization_map, tied_weights_map- remapped = {f"model.{key}": value for key, value in state_dict.items()}- if "model.embed_tokens.weight" in remapped and "lm_head.weight" not in remapped:- remapped["lm_head.weight"] = remapped["model.embed_tokens.weight"]- return remapped, quantization_map, tied_weights_map+ embed_key = "model.embed_tokens.weight"+ if "lm_head.weight" not in state_dict and embed_key in state_dict:+ state_dict["lm_head.weight"] = state_dict[embed_key]+ return state_dict, quantization_map, tied_weights_map++ if self.lm_engine == "vllm":+ from shared.llm_engines.nanovllm.models.qwen3 import Qwen3ForCausalLM as NanoQwen3ForCausalLM+ model_class = NanoQwen3ForCausalLM+ unlimited_budget = True+ else:+ model_class = HfQwen3ForCausalLM+ unlimited_budget = Falseself.lm_model = offload.fast_load_transformers_model(self.lm_weights_path,- modelClass=Qwen3ForCausalLM,+ modelClass=model_class,defaultConfigPath=config_path,default_dtype=self.dtype,preprocess_sd=_remap_lm_state_dict,writable_tensors=False,ignore_unused_weights=True,)+ if unlimited_budget: self.lm_model._budget = 0self.lm_model.eval()- self._disable_lm_compile_for_mmgp()-- def _disable_lm_compile_for_mmgp(self):- if self.lm_model is None:- return- try:- self.lm_model._compile_me = False- for submodule in self.lm_model.modules():- submodule._compile_me = False- except Exception:- return-- def _ensure_lm_module_loaded(self):- if self._lm_create_engine_fn is not None and self._lm_legacy_generate_fn is not None and self._lm_legacy_text_fn is not None:- return- from .qwen3_audio_codes import create_qwen3_lm_engine, generate_audio_codes_legacy, generate_text_legacy-- self._lm_create_engine_fn = create_qwen3_lm_engine- self._lm_legacy_generate_fn = generate_audio_codes_legacy- self._lm_legacy_text_fn = generate_text_legacydef _init_lm_engine(self):- self._ensure_lm_module_loaded()- self._lm_engine_impl = self._lm_create_engine_fn(- engine_name=self.lm_engine,- model=self.lm_model,- tokenizer=self.lm_tokenizer,- device=self.device,- lm_weights_path=self.lm_weights_path,- audio_code_mask=getattr(self, "_audio_code_mask", None),- audio_code_token_map=getattr(self, "_audio_code_token_map", {}),- free_mmap_after_cuda_copy=self.lm_vllm_free_mmap_after_cuda_copy,- weight_load_mode=self.lm_vllm_weight_mode,- )- if self.lm_engine in ("pt", "vllm") and self._lm_engine_impl is None:- raise RuntimeError(- f"Failed to initialize LM engine '{self.lm_engine}'. "- "Check LM weights path and tokenizer availability."- )+ if self.lm_model is None or self.lm_tokenizer is None:+ raise RuntimeError(f"LM engine '{self.lm_engine}' requires loaded LM model and tokenizer.")+ common_kwargs = {"model": self.lm_model, "tokenizer": self.lm_tokenizer, "device": self.device}+ engine_builders = {+ "legacy": lambda: Qwen3LegacyEngine(**common_kwargs),+ "vllm": lambda: Qwen3VllmEngine(lm_weights_path=self.lm_weights_path, **common_kwargs),+ }+ build_engine = engine_builders.get(self.lm_engine, None)+ if build_engine is None:+ raise RuntimeError(f"Unsupported LM engine '{self.lm_engine}'.")+ self._lm_engine_impl = build_engine()def _load_silence_latent(self):if not self.silence_latent_path or not os.path.isfile(self.silence_latent_path):@@ -639,34 +607,11 @@ class ACEStep15Pipeline:stop_checker=None,progress_label: str = "LM text",release_vram_after: bool = True,+ ignore_eos: bool = False,):- if self._lm_engine_impl is not None and hasattr(self._lm_engine_impl, "generate_text"):- return self._lm_engine_impl.generate_text(- prompt=prompt,- prompt_negative=prompt_negative,- max_tokens=max_tokens,- temperature=temperature,- top_p=top_p,- top_k=top_k,- cfg_scale=cfg_scale,- seed=seed,- callback=callback,- abort_fn=self._should_abort,- logits_processor=logits_processor,- logits_processor_update_state=logits_processor_update_state,- stop_checker=stop_checker,- progress_label=progress_label,- release_vram_after=release_vram_after,- )- if self.lm_engine != "legacy":+ if self._lm_engine_impl is None:raise RuntimeError(f"LM engine '{self.lm_engine}' is not initialized for text generation.")- if self.lm_model is None or self.lm_tokenizer is None:- raise RuntimeError("Legacy LM engine requires loaded LM model and tokenizer.")- self._ensure_lm_module_loaded()- return self._lm_legacy_text_fn(- model=self.lm_model,- tokenizer=self.lm_tokenizer,- device=self.device,+ output = self._lm_engine_impl.generate_text(prompt=prompt,prompt_negative=prompt_negative,max_tokens=max_tokens,@@ -681,8 +626,11 @@ class ACEStep15Pipeline:logits_processor_update_state=logits_processor_update_state,stop_checker=stop_checker,progress_label=progress_label,- ignore_eos=False,+ release_vram_after=release_vram_after,+ ignore_eos=ignore_eos,)+ self._lm_last_failure_reason = ""+ return outputdef _ensure_reference_phase1_modules(self):if self._ref_metadata_processor_class is not None:@@ -852,7 +800,9 @@ class ACEStep15Pipeline:release_vram_after=release_vram_after,)if text_out is None:- raise RuntimeError("LM phase1 failed while inferring metadata.")+ return None, None+ if self._should_abort():+ return None, Nonephase1_text = text_out.get("text", "") if isinstance(text_out, dict) else ""parsed_metadata, _ = self._parse_reference_lm_output(phase1_text)@@ -949,8 +899,8 @@ class ACEStep15Pipeline:int(seed_value) if seed_value is not None else None,)- def _reserve_vllm_runtime_for_max_duration(self, tags: str, lyrics: str, negative_prompt: str, lm_cfg_scale):- if self.lm_engine != "vllm" or self._lm_engine_impl is None or self.lm_tokenizer is None:+ def _reserve_lm_runtime_for_max_duration(self, tags: str, lyrics: str, negative_prompt: str, lm_cfg_scale):+ if self._lm_engine_impl is None or self.lm_tokenizer is None:returnreserve_fn = getattr(self._lm_engine_impl, "reserve_runtime", None)if not callable(reserve_fn):@@ -1008,11 +958,6 @@ class ACEStep15Pipeline:):if cfg_scale is None:cfg_scale = 2.5- if self.lm_engine == "vllm" and offloadobj is not None:- try:- offloadobj.unload_all()- except Exception:- passmetadata = {"bpm": bpm,@@ -1029,29 +974,11 @@ class ACEStep15Pipeline:prompt = self._build_lm_prompt_with_cot(tags, lyrics, cot_text, is_negative_prompt=False, negative_prompt=negative_prompt)prompt_negative = self._build_lm_prompt_with_cot(tags, lyrics, cot_text, is_negative_prompt=True, negative_prompt=negative_prompt)- if self._lm_engine_impl is not None:- return self._lm_engine_impl.generate_audio_codes(- prompt=prompt,- prompt_negative=prompt_negative,- min_tokens=min_tokens,- max_tokens=max_tokens,- temperature=temperature,- top_p=top_p,- top_k=top_k,- cfg_scale=cfg_scale,- seed=seed,- callback=callback,- abort_fn=self._should_abort,- )- if self.lm_engine != "legacy":+ if self._lm_engine_impl is None:raise RuntimeError(f"LM engine '{self.lm_engine}' is not initialized.")- if self.lm_model is None or self.lm_tokenizer is None:- raise RuntimeError("Legacy LM engine requires loaded LM model and tokenizer.")- self._ensure_lm_module_loaded()- return self._lm_legacy_generate_fn(- model=self.lm_model,+ audio_codes, failure_reason = generate_audio_codes_with_engine_sampling(+ engine=self._lm_engine_impl,tokenizer=self.lm_tokenizer,- device=self.device,prompt=prompt,prompt_negative=prompt_negative,audio_code_mask=getattr(self, "_audio_code_mask", None),@@ -1065,12 +992,15 @@ class ACEStep15Pipeline:seed=seed,callback=callback,abort_fn=self._should_abort,+ release_vram_after=True,)+ self._lm_last_failure_reason = failure_reason+ return audio_codes- def _release_vllm_lm_runtime(self):- if self.lm_engine != "vllm" or self._lm_engine_impl is None:+ def _release_lm_runtime(self):+ if self._lm_engine_impl is None:return- release_fn = getattr(self._lm_engine_impl, "release_vram", None)+ release_fn = getattr(self._lm_engine_impl, "release_runtime_allocations", None)if not callable(release_fn):returntry:@@ -1233,9 +1163,9 @@ class ACEStep15Pipeline:progress_unit="tokens",)if torch.is_tensor(cached):- self._release_vllm_lm_runtime()+ self._release_lm_runtime()return cached.tolist()- self._release_vllm_lm_runtime()+ self._release_lm_runtime()return list(cached)def _default_timbre_latents(self, length):@@ -1882,7 +1812,12 @@ class ACEStep15Pipeline:elif isinstance(user_audio_codes, (list, tuple)) and user_audio_codes and isinstance(user_audio_codes[0], str):parsed = self._parse_audio_code_string(user_audio_codes[0])user_audio_codes = parsed if parsed else None- keep_vllm_lm_loaded_for_phase2 = bool(self.enable_lm and self.lm_engine == "vllm" and (not has_src_audio) and (not user_audio_codes))+ keep_lm_loaded_for_phase2 = bool(+ self.enable_lm+ and (not has_src_audio)+ and (not user_audio_codes)+ and bool(getattr(self._lm_engine_impl, "keep_loaded_for_phase2", False))+ )phase1_metadata = {"bpm": user_bpm,@@ -1890,163 +1825,169 @@ class ACEStep15Pipeline:"timesignature": user_timesignature,"language": user_language,}- if self.enable_lm:- try:- self._reserve_vllm_runtime_for_max_duration(tags, lyrics, lm_negative_prompt, lm_cfg_scale)- except Exception:- pass- tags_for_generation = tags- computed_phase1_metadata = {}- refined_caption = None- if model_mode_value == _ACE_STEP15_MODEL_MODE_DEFAULT:- bpm = phase1_metadata["bpm"] if phase1_metadata["bpm"] is not None else _ACE_STEP15_DEFAULT_BPM- timesignature = phase1_metadata["timesignature"] if phase1_metadata["timesignature"] is not None else _ACE_STEP15_DEFAULT_TIMESIGNATURE- if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:- timesignature = _ACE_STEP15_DEFAULT_TIMESIGNATURE- keyscale = phase1_metadata["keyscale"] if phase1_metadata["keyscale"] is not None else _ACE_STEP15_DEFAULT_KEYSCALE- keyscale = str(keyscale).strip()- if len(keyscale) == 0:- keyscale = _ACE_STEP15_DEFAULT_KEYSCALE- language = phase1_metadata["language"] if phase1_metadata["language"] is not None else "en"- else:- if not self.enable_lm:- raise RuntimeError("ACE-Step 1.5 model mode 1/2/3 requires an LM definition (text_encoder_URLs).")- missing_fields = [field_name for field_name in ("bpm", "keyscale", "timesignature", "language") if phase1_metadata.get(field_name) is None]- infer_language = phase1_metadata.get("language", None) is None- run_phase1 = model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION) or (- model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_MISSING and len(missing_fields) > 0- )- if run_phase1:- if self.lm_engine == "vllm" and offloadobj is not None:- try:- offloadobj.unload_all()- except Exception:- pass- phase1_seed = seed if seed is not None else 0- _update_progress_status("LM Compute Metadata")- phase1_metadata, refined_caption = self._run_phase1_metadata(- tags=tags,+ lm_runtime_retained = keep_lm_loaded_for_phase2+ try:+ if self.enable_lm:+ try:+ self._reserve_lm_runtime_for_max_duration(tags, lyrics, lm_negative_prompt, lm_cfg_scale)+ except Exception:+ pass+ tags_for_generation = tags+ computed_phase1_metadata = {}+ refined_caption = None+ if model_mode_value == _ACE_STEP15_MODEL_MODE_DEFAULT:+ bpm = phase1_metadata["bpm"] if phase1_metadata["bpm"] is not None else _ACE_STEP15_DEFAULT_BPM+ timesignature = phase1_metadata["timesignature"] if phase1_metadata["timesignature"] is not None else _ACE_STEP15_DEFAULT_TIMESIGNATURE+ if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:+ timesignature = _ACE_STEP15_DEFAULT_TIMESIGNATURE+ keyscale = phase1_metadata["keyscale"] if phase1_metadata["keyscale"] is not None else _ACE_STEP15_DEFAULT_KEYSCALE+ keyscale = str(keyscale).strip()+ if len(keyscale) == 0:+ keyscale = _ACE_STEP15_DEFAULT_KEYSCALE+ language = phase1_metadata["language"] if phase1_metadata["language"] is not None else "en"+ else:+ if not self.enable_lm:+ raise RuntimeError("ACE-Step 1.5 model mode 1/2/3 requires an LM definition (text_encoder_URLs).")+ missing_fields = [field_name for field_name in ("bpm", "keyscale", "timesignature", "language") if phase1_metadata.get(field_name) is None]+ infer_language = phase1_metadata.get("language", None) is None+ run_phase1 = model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION) or (+ model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_MISSING and len(missing_fields) > 0+ )+ if run_phase1:+ phase1_seed = seed if seed is not None else 0+ _update_progress_status("LM Compute Metadata")+ phase1_metadata, refined_caption = self._run_phase1_metadata(+ tags=tags,+ lyrics=lyrics,+ metadata=phase1_metadata,+ refine_caption=model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION),+ infer_language=infer_language,+ seed=phase1_seed,+ cfg_scale=1.0 if lm_cfg_scale is None else float(lm_cfg_scale),+ negative_prompt=lm_negative_prompt,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ callback=_phase_callback,+ release_vram_after=not keep_lm_loaded_for_phase2,+ )+ for field_name in missing_fields:+ value = phase1_metadata.get(field_name, None) if phase1_metadata is not None else None+ if value is not None:+ computed_phase1_metadata[field_name] = value+ if phase1_metadata is None:+ return None+ if model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION:+ duration_value = self._parse_optional_int_custom_setting(phase1_metadata.get("duration", None))+ if duration_value is None:+ raise RuntimeError("LM phase1 failed to resolve required metadata 'duration'.")+ duration_value = max(_ACE_STEP15_DURATION_MIN_SECONDS, min(_ACE_STEP15_DURATION_MAX_SECONDS, int(duration_value)))+ duration_seconds = float(duration_value)+ computed_phase1_metadata["duration"] = int(duration_value)+ if len(computed_phase1_metadata) > 0:+ print(f"[ace_step15][phase1] computed metadata: {computed_phase1_metadata}")+ if refined_caption is not None:+ tags_for_generation = refined_caption+ if refined_caption.strip() != tags.strip():+ print(f"[ace_step15][phase1] refined caption: {refined_caption}")+ if set_header_text is not None:+ header_parts = []+ if len(computed_phase1_metadata) > 0:+ computed_bits = []+ for field_name in ("bpm", "keyscale", "timesignature", "language", "duration"):+ if field_name in computed_phase1_metadata:+ computed_bits.append(f"{field_name}={computed_phase1_metadata[field_name]}")+ if len(computed_bits) > 0:+ header_parts.append("LM computed metadata: " + "<BR>- ".join(computed_bits))+ if refined_caption is not None and refined_caption.strip() != tags.strip():+ header_parts.append(f"<BR>- LM refined caption: {refined_caption}")+ if len(header_parts) > 0:+ try:+ set_header_text("".join(header_parts))+ except Exception:+ pass+ for field_name in ("bpm", "keyscale", "timesignature", "language"):+ if phase1_metadata.get(field_name, None) is None:+ raise RuntimeError(f"LM phase1 failed to resolve required metadata '{field_name}'.")+ bpm = int(phase1_metadata["bpm"])+ timesignature = int(phase1_metadata["timesignature"])+ if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:+ raise RuntimeError("LM phase1 produced an unsupported timesignature.")+ keyscale = str(phase1_metadata["keyscale"]).strip()+ if len(keyscale) == 0:+ raise RuntimeError("LM phase1 produced an empty keyscale.")+ language = str(phase1_metadata["language"]).strip().lower()+ if len(language) == 0:+ raise RuntimeError("LM phase1 produced an empty language.")+ if language not in _ACE_STEP15_VALID_LANGUAGE_SET:+ raise RuntimeError(f"LM phase1 produced unsupported language '{language}'.")++ duration_int = int(math.ceil(duration_seconds))+ min_tokens = duration_int * 5++ meta_cap = self._format_meta(bpm, duration_int, keyscale, timesignature)+ use_timbre = "B" in (audio_prompt_type or "")++ audio_codes = None+ if user_audio_codes:+ max_code = getattr(self, "_audio_code_max", None)+ if max_code is not None:+ bad_codes = [v for v in user_audio_codes if v < 0 or v > max_code]+ if bad_codes:+ raise ValueError(f"Audio codes out of range 0..{max_code}; example={bad_codes[0]}")+ audio_codes = user_audio_codes+ elif self.enable_lm and not has_src_audio:+ _update_progress_status("LM Compute Audio Codes")+ audio_codes = self._generate_audio_codes(+ tags=tags_for_generation,lyrics=lyrics,- metadata=phase1_metadata,- refine_caption=model_mode_value in (_ACE_STEP15_MODEL_MODE_INFER_AND_REFINE, _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION),- infer_language=infer_language,- seed=phase1_seed,- cfg_scale=1.0 if lm_cfg_scale is None else float(lm_cfg_scale),- negative_prompt=lm_negative_prompt,+ bpm=bpm,+ duration=duration_int,+ keyscale=keyscale,+ timesignature=timesignature,+ seed=seed if seed is not None else 0,+ min_tokens=min_tokens,+ max_tokens=min_tokens,temperature=temperature,top_p=top_p,top_k=top_k,+ language=language,+ negative_prompt=lm_negative_prompt,+ cfg_scale=lm_cfg_scale,callback=_phase_callback,- release_vram_after=not keep_vllm_lm_loaded_for_phase2,- )- for field_name in missing_fields:- value = phase1_metadata.get(field_name, None) if phase1_metadata is not None else None- if value is not None:- computed_phase1_metadata[field_name] = value- if phase1_metadata is None:- return None- if model_mode_value == _ACE_STEP15_MODEL_MODE_INFER_REFINE_AND_DURATION:- duration_value = self._parse_optional_int_custom_setting(phase1_metadata.get("duration", None))- if duration_value is None:- raise RuntimeError("LM phase1 failed to resolve required metadata 'duration'.")- duration_value = max(_ACE_STEP15_DURATION_MIN_SECONDS, min(_ACE_STEP15_DURATION_MAX_SECONDS, int(duration_value)))- duration_seconds = float(duration_value)- computed_phase1_metadata["duration"] = int(duration_value)- if len(computed_phase1_metadata) > 0:- print(f"[ace_step15][phase1] computed metadata: {computed_phase1_metadata}")- if refined_caption is not None:- tags_for_generation = refined_caption- if refined_caption.strip() != tags.strip():- print(f"[ace_step15][phase1] refined caption: {refined_caption}")- if set_header_text is not None:- header_parts = []- if len(computed_phase1_metadata) > 0:- computed_bits = []- for field_name in ("bpm", "keyscale", "timesignature", "language", "duration"):- if field_name in computed_phase1_metadata:- computed_bits.append(f"{field_name}={computed_phase1_metadata[field_name]}")- if len(computed_bits) > 0:- header_parts.append("LM computed metadata: " + "<BR>- ".join(computed_bits))- if refined_caption is not None and refined_caption.strip() != tags.strip():- header_parts.append(f"<BR>- LM refined caption: {refined_caption}")- if len(header_parts) > 0:- try:- set_header_text("".join(header_parts))- except Exception:- pass- for field_name in ("bpm", "keyscale", "timesignature", "language"):- if phase1_metadata.get(field_name, None) is None:- raise RuntimeError(f"LM phase1 failed to resolve required metadata '{field_name}'.")- bpm = int(phase1_metadata["bpm"])- timesignature = int(phase1_metadata["timesignature"])- if timesignature not in _ACE_STEP15_ALLOWED_TIMESIGNATURES:- raise RuntimeError("LM phase1 produced an unsupported timesignature.")- keyscale = str(phase1_metadata["keyscale"]).strip()- if len(keyscale) == 0:- raise RuntimeError("LM phase1 produced an empty keyscale.")- language = str(phase1_metadata["language"]).strip().lower()- if len(language) == 0:- raise RuntimeError("LM phase1 produced an empty language.")- if language not in _ACE_STEP15_VALID_LANGUAGE_SET:- raise RuntimeError(f"LM phase1 produced unsupported language '{language}'.")-- duration_int = int(math.ceil(duration_seconds))- min_tokens = duration_int * 5-- meta_cap = self._format_meta(bpm, duration_int, keyscale, timesignature)- use_timbre = "B" in (audio_prompt_type or "")-- audio_codes = None- if user_audio_codes:- max_code = getattr(self, "_audio_code_max", None)- if max_code is not None:- bad_codes = [v for v in user_audio_codes if v < 0 or v > max_code]- if bad_codes:- raise ValueError(f"Audio codes out of range 0..{max_code}; example={bad_codes[0]}")- audio_codes = user_audio_codes- elif self.enable_lm and not has_src_audio:- _update_progress_status("LM Compute Audio Codes")- audio_codes = self._generate_audio_codes(- tags=tags_for_generation,- lyrics=lyrics,- bpm=bpm,- duration=duration_int,- keyscale=keyscale,- timesignature=timesignature,- seed=seed if seed is not None else 0,- min_tokens=min_tokens,- max_tokens=min_tokens,- temperature=temperature,- top_p=top_p,- top_k=top_k,- language=language,- negative_prompt=lm_negative_prompt,- cfg_scale=lm_cfg_scale,- callback=_phase_callback,- offloadobj=offloadobj,- )- if audio_codes is None:- return None- if len(audio_codes) == 0:- failure_reason = ""- if self._lm_engine_impl is not None:- get_reason = getattr(self._lm_engine_impl, "get_last_failure_reason", None)- if callable(get_reason):- try:- failure_reason = str(get_reason() or "")- except Exception:- failure_reason = ""- extra = f" last_failure={failure_reason}" if failure_reason else ""- raise RuntimeError(- "Audio code generation returned 0 usable codes "- f"(engine={self.lm_engine}, tokenizer_dir='{self.lm_tokenizer_dir}', "- f"audio_code_vocab={len(getattr(self, '_audio_code_token_ids', []))}).{extra}"+ offloadobj=offloadobj,)- max_code = getattr(self, "_audio_code_max", None)- if audio_codes is not None and max_code is not None:- bad_codes = [v for v in audio_codes if v < 0 or v > max_code]- if bad_codes:- raise RuntimeError(f"LM generated out-of-range audio codes; example={bad_codes[0]}")+ if audio_codes is None:+ return None+ if self._should_abort():+ return None+ if len(audio_codes) == 0:+ if self._should_abort():+ return None+ failure_reason = str(getattr(self, "_lm_last_failure_reason", "") or "")+ if self._lm_engine_impl is not None:+ get_reason = getattr(self._lm_engine_impl, "get_last_failure_reason", None)+ if callable(get_reason):+ try:+ engine_reason = str(get_reason() or "")+ if len(engine_reason) > 0:+ failure_reason = engine_reason+ except Exception:+ pass+ extra = f" last_failure={failure_reason}" if failure_reason else ""+ raise RuntimeError(+ "Audio code generation returned 0 usable codes "+ f"(engine={self.lm_engine}, tokenizer_dir='{self.lm_tokenizer_dir}', "+ f"audio_code_vocab={len(getattr(self, '_audio_code_token_ids', []))}).{extra}"+ )+ max_code = getattr(self, "_audio_code_max", None)+ if audio_codes is not None and max_code is not None:+ bad_codes = [v for v in audio_codes if v < 0 or v > max_code]+ if bad_codes:+ raise RuntimeError(f"LM generated out-of-range audio codes; example={bad_codes[0]}")+ finally:+ if lm_runtime_retained:+ self._release_lm_runtime()_update_progress_status("")if audio_codes is not None:diff --git a/models/TTS/ace_step15/qwen3_audio_codes.py b/models/TTS/ace_step15/qwen3_audio_codes.pyindex f1c209b..a638c5f 100644--- a/models/TTS/ace_step15/qwen3_audio_codes.py+++ b/models/TTS/ace_step15/qwen3_audio_codes.py@@ -1,10 +1,8 @@-"""Qwen3 LM audio-code decoding engines (legacy/pt/vllm)."""+"""Qwen3 LM audio-code sampling helpers (engine-neutral)."""from __future__ import annotations-import osimport re-import sysfrom typing import Any, Callable, Optionalimport torch@@ -82,40 +80,6 @@ def _postprocess_audio_codes(codes, min_tokens: int, max_tokens: int):codes.extend([pad_val] * (min_tokens - len(codes)))return codes[:max_tokens]--def _guess_lm_model_size(model_path: str) -> str | None:- name = (model_path or "").lower()- if "0.6b" in name:- return "0.6B"- if "1.7b" in name:- return "1.7B"- if "4b" in name:- return "4B"- return None---def _compute_vllm_gpu_utilization(model_path: str) -> float:- try:- if not torch.cuda.is_available():- return 0.9- total_gb = torch.cuda.get_device_properties(0).total_memory / (1024 ** 3)- except Exception:- return 0.9- if total_gb <= 0:- return 0.9- target_map = {- "0.6B": 3.0,- "1.7B": 8.0,- "4B": 12.0,- }- model_size = _guess_lm_model_size(model_path)- target_gb = target_map.get(model_size, 3.0)- ratio = target_gb / total_gb- if total_gb >= 24:- return min(0.9, max(0.2, ratio))- return min(0.9, max(0.1, ratio))--class AudioCodeMaskProcessor(LogitsProcessor):def __init__(self, audio_code_mask: Optional[torch.Tensor]):self._mask_cpu = audio_code_mask@@ -327,7 +291,7 @@ def _generate_token_ids_legacy(return token_ids-def generate_text_legacy(+def generate_text_sampling(model,tokenizer,device,@@ -373,10 +337,9 @@ def generate_text_legacy(return {"token_ids": token_ids, "text": text}-def generate_audio_codes_legacy(- model,+def generate_audio_codes_with_engine_sampling(+ engine,tokenizer,- device,prompt: str,prompt_negative: str,audio_code_mask: Optional[torch.Tensor],@@ -390,12 +353,10 @@ def generate_audio_codes_legacy(seed: Optional[int],callback=None,abort_fn: Optional[Callable[[], bool]] = None,+ release_vram_after: bool = True,):mask_processor = AudioCodeMaskProcessor(audio_code_mask)- text_out = generate_text_legacy(- model=model,- tokenizer=tokenizer,- device=device,+ text_out = engine.generate_text(prompt=prompt,prompt_negative=prompt_negative,max_tokens=max_tokens,@@ -407,671 +368,64 @@ def generate_audio_codes_legacy(callback=callback,abort_fn=abort_fn,logits_processor=mask_processor,+ logits_processor_update_state=None,+ stop_checker=None,progress_label="Compute Audio Codes",+ release_vram_after=release_vram_after,ignore_eos=True,)if text_out is None:- return None+ return None, ""+ if abort_fn is not None and abort_fn():+ return None, ""++ if isinstance(text_out, dict):+ token_ids = text_out.get("token_ids", []) or []+ decoded_text = str(text_out.get("text", "") or "")+ else:+ token_ids = []+ decoded_text = str(text_out or "")+ token_ids = [int(x) for x in token_ids]++ parsed_codes = []+ unmapped_count = 0+ out_of_vocab_count = 0+ vocab_size = None+ try:+ vocab_size = len(tokenizer.get_vocab())+ except Exception:+ vocab_size = None- token_ids = text_out.get("token_ids", [])- audio_codes = []for token_id in token_ids:- code_val = _token_id_to_audio_code(token_id, audio_code_token_map, tokenizer)+ code_val = _token_id_to_audio_code(int(token_id), audio_code_token_map, tokenizer)if code_val is not None:- audio_codes.append(code_val)- if len(audio_codes) >= max_tokens:+ parsed_codes.append(code_val)+ if len(parsed_codes) >= max_tokens:break+ else:+ unmapped_count += 1+ if vocab_size is not None and (token_id < 0 or token_id >= vocab_size):+ out_of_vocab_count += 1- return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)---class Qwen3PtEngine:- def __init__(- self,- model,- tokenizer,- device,- audio_code_mask: Optional[torch.Tensor],- audio_code_token_map: dict[int, int],- ):- self.model = model- self.tokenizer = tokenizer- self.device = device- self.audio_code_mask = audio_code_mask- self.audio_code_token_map = audio_code_token_map- self._mask_processor = AudioCodeMaskProcessor(audio_code_mask)-- def generate_audio_codes(- self,- prompt: str,- prompt_negative: str,- min_tokens: int,- max_tokens: int,- temperature: Optional[float],- top_p: Optional[float],- top_k: Optional[int],- cfg_scale: float,- seed: Optional[int],- callback=None,- abort_fn: Optional[Callable[[], bool]] = None,- ):- if cfg_scale > 1.0:- return generate_audio_codes_legacy(- self.model,- self.tokenizer,- self.device,- prompt,- prompt_negative,- self.audio_code_mask,- self.audio_code_token_map,- min_tokens,- max_tokens,- temperature,- top_p,- top_k,- cfg_scale,- seed,- callback=callback,- abort_fn=abort_fn,- )-- if abort_fn is not None and abort_fn():- return None-- inputs = self.tokenizer(prompt, return_tensors="pt", add_special_tokens=False)- input_ids = inputs["input_ids"].to(self.device)- attention_mask = inputs.get("attention_mask")- if attention_mask is not None:- attention_mask = attention_mask.to(self.device)-- generator = None- if seed is not None and seed >= 0:- generator = torch.Generator(device=self.device)- generator.manual_seed(seed)-- logits_processors = LogitsProcessorList([self._mask_processor])-- if callback is not None:- callback(- step_idx=-1,- override_num_inference_steps=max_tokens,- denoising_extra=f"Compute Audio Codes 0/{max_tokens}",- progress_unit="tokens",- )-- do_sample = temperature is not None and temperature > 0- with torch.no_grad():- outputs = self.model.generate(- input_ids=input_ids,- attention_mask=attention_mask,- max_new_tokens=max_tokens,- do_sample=do_sample,- temperature=temperature if do_sample else 1.0,- top_k=top_k if top_k is not None and top_k > 0 else None,- top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,- logits_processor=logits_processors,- pad_token_id=self.tokenizer.pad_token_id or self.tokenizer.eos_token_id,- generator=generator,- )-- generated_ids = outputs[0][input_ids.shape[1]:]- if generated_ids.is_cuda:- generated_ids = generated_ids.cpu()-- audio_codes = []- for token_id in generated_ids.tolist():- code_val = _token_id_to_audio_code(int(token_id), self.audio_code_token_map, self.tokenizer)- if code_val is not None:- audio_codes.append(code_val)- if len(audio_codes) >= max_tokens:- break-- if callback is not None:- callback(- step_idx=max(0, max_tokens - 1),- override_num_inference_steps=max_tokens,- denoising_extra=f"Compute Audio Codes {max_tokens}/{max_tokens}",- progress_unit="tokens",- )-- return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)-- def generate_text(- self,- prompt: str,- prompt_negative: str,- max_tokens: int,- temperature: Optional[float],- top_p: Optional[float],- top_k: Optional[int],- cfg_scale: float,- seed: Optional[int],- callback=None,- abort_fn: Optional[Callable[[], bool]] = None,- logits_processor: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None,- logits_processor_update_state: Optional[Callable[[int], None]] = None,- stop_checker: Optional[Callable[[list[int], int], bool]] = None,- progress_label: str = "LM text",- release_vram_after: bool = True,- ):- del release_vram_after- return generate_text_legacy(- model=self.model,- tokenizer=self.tokenizer,- device=self.device,- prompt=prompt,- prompt_negative=prompt_negative,- max_tokens=max_tokens,- temperature=temperature,- top_p=top_p,- top_k=top_k,- cfg_scale=cfg_scale,- seed=seed,- callback=callback,- abort_fn=abort_fn,- logits_processor=logits_processor,- logits_processor_update_state=logits_processor_update_state,- stop_checker=stop_checker,- progress_label=progress_label,- ignore_eos=False,- )---class Qwen3VllmEngine:- def __init__(- self,- lm_weights_path: str,- tokenizer,- audio_code_mask: Optional[torch.Tensor],- audio_code_token_map: dict[int, int],- free_mmap_after_cuda_copy: bool = True,- weight_load_mode: str = "lazy",- ):- self.lm_weights_path = lm_weights_path- self.tokenizer = tokenizer- self.audio_code_mask = audio_code_mask- self.audio_code_token_map = audio_code_token_map- self.free_mmap_after_cuda_copy = free_mmap_after_cuda_copy- self._llm = None- self._sampling_params_cls = None- self._model_dir = None- self._max_model_len_hint = None- self._max_num_seqs_hint = None- self._max_num_batched_tokens_hint = None- self._last_zero_code_diagnostic = None- self.weight_load_mode = (weight_load_mode or "lazy").strip().lower()- if self.weight_load_mode not in ("lazy", "pinned"):- raise ValueError(f"Unsupported vllm weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")-- def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):- req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(- prompt_len=prompt_len,- max_tokens=max_tokens,- cfg_scale=cfg_scale,- )- self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)-- def _release_vram(self):- if self._llm is None:- return- try:- self._llm.unload_weights()- except Exception:- pass-- def release_vram(self):- self._release_vram()-- @staticmethod- def _compute_runtime_hints(prompt_len: int, max_tokens: int, cfg_scale: float):- max_model_len = max(8, int(prompt_len) + int(max_tokens))- max_num_seqs = 2 if cfg_scale and cfg_scale > 1.0 else 1- max_num_batched_tokens = max_model_len * max_num_seqs- return max_model_len, max_num_seqs, max_num_batched_tokens-- def _ensure_runtime_capacity(self, max_model_len: int, max_num_seqs: int, max_num_batched_tokens: int):- if self._max_model_len_hint is None:- self._max_model_len_hint = max_model_len- self._max_num_seqs_hint = max_num_seqs- self._max_num_batched_tokens_hint = max_num_batched_tokens- return-- need_grow = (- max_model_len > int(self._max_model_len_hint)- or max_num_seqs > int(self._max_num_seqs_hint)- or max_num_batched_tokens > int(self._max_num_batched_tokens_hint)- )- if not need_grow:- return-- self._max_model_len_hint = max_model_len- self._max_num_seqs_hint = max_num_seqs- self._max_num_batched_tokens_hint = max_num_batched_tokens- # Existing LLM was built with smaller runtime limits; rebuild safely.- self.close()-- def _ensure_nanovllm_on_path(self):- llm_engines_root = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "shared", "llm_engines"))- if llm_engines_root not in sys.path:- sys.path.insert(0, llm_engines_root)-- def _prepare_model_dir(self):- if self._model_dir is not None:- return self._model_dir-- if not self.lm_weights_path:- raise RuntimeError("vllm engine requires lm_weights_path")-- # For vLLM/nano-vllm, we can point directly at the weight file.- # nanovllm.config will derive model_dir from the file path to find config/tokenizer.- self._model_dir = self.lm_weights_path- return self._model_dir-- def _ensure_llm(self):- if self._llm is not None:- return- # Keep online-softmax path enabled for compiled LM sampling kernels.- # Without this, Inductor may split reductions and emit a warning while- # falling back to a slower softmax lowering.- try:- import torch._inductor.config as inductor_config-- if bool(getattr(inductor_config, "split_reductions", False)):- inductor_config.split_reductions = False- except Exception:- pass- self._ensure_nanovllm_on_path()- try:- from nanovllm import LLM, SamplingParams- except Exception as exc: # pragma: no cover- raise RuntimeError(f"nano-vllm is not available for vllm engine: {exc}") from exc-- model_dir = self._prepare_model_dir()- gpu_memory_utilization = _compute_vllm_gpu_utilization(self.lm_weights_path)- max_model_len = self._max_model_len_hint or 4096- max_num_seqs = self._max_num_seqs_hint or 1- max_num_batched_tokens = self._max_num_batched_tokens_hint or (max_model_len * max_num_seqs)- self._llm = LLM(- model=model_dir,- enforce_eager=False,- tensor_parallel_size=1,- max_model_len=max_model_len,- max_num_seqs=max_num_seqs,- max_num_batched_tokens=max_num_batched_tokens,- gpu_memory_utilization=gpu_memory_utilization,- tokenizer=self.tokenizer,- free_mmap_after_cuda_copy=self.free_mmap_after_cuda_copy,- weight_load_mode=self.weight_load_mode,- )- self._sampling_params_cls = SamplingParams-- def close(self):- llm = getattr(self, "_llm", None)- self._llm = None- if llm is not None:- try:- close_fn = getattr(llm, "close", None)- if callable(close_fn):- close_fn()- else:- try:- llm.unload_weights()- except Exception:- pass- try:- llm.clear_graph_cache()- except Exception:- pass- exit_fn = getattr(llm, "exit", None)- if callable(exit_fn):- exit_fn()- except Exception:- pass- try:- del llm- except Exception:- pass- # Keep close() idempotent; this method is also used for runtime rebuilds.- self._sampling_params_cls = None- self._model_dir = None-- def get_last_failure_reason(self) -> str:- diag = self._last_zero_code_diagnostic- if not diag:- return ""- return (- f"attempt={diag.get('attempt')} token_ids={diag.get('token_ids')} "- f"unmapped={diag.get('unmapped')} out_of_vocab={diag.get('out_of_vocab')} "- f"regex_hits={diag.get('regex_hits')} first_token_ids={diag.get('first_token_ids')} "- f"text_preview='{diag.get('text_preview')}'"- )-- def __del__(self):- self.close()-- @staticmethod- def _extract_text_and_tokens(output_obj) -> tuple[str, list[int]]:- if output_obj is None:- return "", []- if isinstance(output_obj, dict):- text = str(output_obj.get("text", "") or "")- token_ids = output_obj.get("token_ids", []) or []- return text, [int(x) for x in token_ids]- if hasattr(output_obj, "outputs"):- outputs = getattr(output_obj, "outputs", None)- if outputs and len(outputs) > 0:- text = str(getattr(outputs[0], "text", "") or "")- token_ids = getattr(outputs[0], "token_ids", None)- if token_ids is None:- token_ids = getattr(outputs[0], "token_ids_list", []) or []- return text, [int(x) for x in token_ids] if token_ids else []- text = str(getattr(output_obj, "text", "") or "")- token_ids = getattr(output_obj, "token_ids", None)- if token_ids is None:- token_ids = []- return text, [int(x) for x in token_ids]-- def generate_audio_codes(- self,- prompt: str,- prompt_negative: str,- min_tokens: int,- max_tokens: int,- temperature: Optional[float],- top_p: Optional[float],- top_k: Optional[int],- cfg_scale: float,- seed: Optional[int],- callback=None,- abort_fn: Optional[Callable[[], bool]] = None,- ):- if abort_fn is not None and abort_fn():- return None-+ regex_hits = 0+ if len(parsed_codes) == 0 and decoded_text:try:- prompt_len = len(self.tokenizer.encode(prompt))+ parsed_codes = [int(x) for x in _AUDIO_CODE_RE.findall(decoded_text)]+ regex_hits = len(parsed_codes)+ if len(parsed_codes) > max_tokens:+ parsed_codes = parsed_codes[:max_tokens]except Exception:- prompt_len = 0- # CFG runs conditional + unconditional sequences; runtime capacity must fit- # the longest prompt between both or decode can enter invalid states.- if cfg_scale > 1.0 and prompt_negative:- try:- prompt_len = max(prompt_len, len(self.tokenizer.encode(prompt_negative)))- except Exception:- pass- req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(- prompt_len=prompt_len,- max_tokens=max_tokens,- cfg_scale=cfg_scale,- )- self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)- self._ensure_llm()- if self._llm is None:- return None- try:- self._llm.reset()- except Exception:- pass-- if callback is not None:- callback(- step_idx=-1,- override_num_inference_steps=max_tokens,- denoising_extra=f"Compute Audio Codes 0/{max_tokens}",- progress_unit="tokens",- )-- seed_value = None- if seed is not None:- try:- seed_value = int(seed)- except Exception:- seed_value = None- if seed_value is not None and seed_value < 0:- seed_value = None-- temp = temperature if temperature is not None and temperature > 0 else 1e-5- sampling_params = self._sampling_params_cls(- temperature=temp,- max_tokens=max_tokens,- cfg_scale=max(cfg_scale, 1.0),- top_k=top_k if top_k is not None and top_k > 0 else None,- top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,- ignore_eos=True,- logits_processor=AudioCodeMaskProcessor(self.audio_code_mask),- seed=seed_value,- )-- audio_codes = []- self._last_zero_code_diagnostic = None- for attempt in range(2):- outputs = self._llm.generate(- prompts=[prompt],- sampling_params=sampling_params,- use_tqdm=True,- unconditional_prompts=[prompt_negative] if cfg_scale > 1.0 else None,- )-- token_ids = outputs[0].get("token_ids", []) if outputs else []parsed_codes = []- unmapped_count = 0- out_of_vocab_count = 0- vocab_size = None- try:- vocab_size = len(self.tokenizer.get_vocab())- except Exception:- vocab_size = None- for token_id in token_ids:- code_val = _token_id_to_audio_code(int(token_id), self.audio_code_token_map, self.tokenizer)- if code_val is not None:- parsed_codes.append(code_val)- if len(parsed_codes) >= max_tokens:- break- else:- unmapped_count += 1- if vocab_size is not None and (token_id < 0 or token_id >= vocab_size):- out_of_vocab_count += 1-- # Fallback parser: extract <|audio_code_x|> from decoded text if token_ids couldn't be mapped.- if len(parsed_codes) == 0 and outputs:- try:- decoded_text = str(outputs[0].get("text", ""))- parsed_codes = [int(x) for x in _AUDIO_CODE_RE.findall(decoded_text)]- if len(parsed_codes) > max_tokens:- parsed_codes = parsed_codes[:max_tokens]- except Exception:- parsed_codes = []-- if parsed_codes:- audio_codes = parsed_codes- self._last_zero_code_diagnostic = None- break-- decoded_text = ""regex_hits = 0- if outputs:- try:- decoded_text = str(outputs[0].get("text", ""))- except Exception:- decoded_text = ""- if decoded_text:- try:- regex_hits = len(_AUDIO_CODE_RE.findall(decoded_text))- except Exception:- regex_hits = 0- text_preview = decoded_text[:280].replace("\n", "\\n")- self._last_zero_code_diagnostic = {- "attempt": attempt + 1,- "token_ids": len(token_ids),- "unmapped": unmapped_count,- "out_of_vocab": out_of_vocab_count,- "regex_hits": regex_hits,- "first_token_ids": token_ids[:24],- "text_preview": text_preview,- }- print(- f"[ace_step15][vllm][diagnostic] attempt={attempt+1} "- f"token_ids={len(token_ids)} parsed_codes=0 "- f"unmapped={unmapped_count} out_of_vocab={out_of_vocab_count} "- f"first_token_ids={token_ids[:24]} text_preview='{text_preview}'"- )-- if attempt == 0:- print("[ace_step15] Warning: vllm LM returned 0 audio codes; rebuilding LM runtime and retrying once.")- self.close()- self._ensure_llm()- if self._llm is None:- break-- if callback is not None:- callback(- step_idx=max(0, max_tokens - 1),- override_num_inference_steps=max_tokens,- denoising_extra=f"Compute Audio Codes {max_tokens}/{max_tokens}",- progress_unit="tokens",- )-- self._release_vram()-- return _postprocess_audio_codes(audio_codes, min_tokens, max_tokens)-- def generate_text(- self,- prompt: str,- prompt_negative: str,- max_tokens: int,- temperature: Optional[float],- top_p: Optional[float],- top_k: Optional[int],- cfg_scale: float,- seed: Optional[int],- callback=None,- abort_fn: Optional[Callable[[], bool]] = None,- logits_processor: Optional[Any] = None,- logits_processor_update_state: Optional[Callable[[int], None]] = None,- stop_checker: Optional[Callable[[list[int], int], bool]] = None,- progress_label: str = "LM text",- release_vram_after: bool = True,- ):- del stop_checker # nanovllm does not support external stop callbacks yet.- if abort_fn is not None and abort_fn():- return None-- try:- prompt_len = len(self.tokenizer.encode(prompt))- except Exception:- prompt_len = 0- if cfg_scale > 1.0 and prompt_negative:- try:- prompt_len = max(prompt_len, len(self.tokenizer.encode(prompt_negative)))- except Exception:- pass-- req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(- prompt_len=prompt_len,- max_tokens=max_tokens,- cfg_scale=cfg_scale,- )- self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)- self._ensure_llm()- if self._llm is None:- return None- try:- self._llm.reset()- except Exception:- pass-- if callback is not None:- callback(- step_idx=-1,- override_num_inference_steps=max_tokens,- denoising_extra=f"{progress_label} 0/{max_tokens}",- progress_unit="tokens",- )-- seed_value = None- if seed is not None:- try:- seed_value = int(seed)- except Exception:- seed_value = None- if seed_value is not None and seed_value < 0:- seed_value = None-- temp = temperature if temperature is not None and temperature > 0 else 1e-5- sampling_params = self._sampling_params_cls(- temperature=temp,- max_tokens=max_tokens,- cfg_scale=max(cfg_scale, 1.0),- top_k=top_k if top_k is not None and top_k > 0 else None,- top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,- ignore_eos=False,- logits_processor=logits_processor,- logits_processor_update_state=logits_processor_update_state,- seed=seed_value,- )-- outputs = self._llm.generate(- prompts=[prompt],- sampling_params=sampling_params,- use_tqdm=True,- unconditional_prompts=[prompt_negative] if cfg_scale > 1.0 else None,+ if abort_fn is not None and abort_fn():+ return None, ""++ failure_reason = ""+ if len(parsed_codes) == 0:+ text_preview = decoded_text[:280].replace("\n", "\\n")+ failure_reason = (+ f"token_ids={len(token_ids)} unmapped={unmapped_count} out_of_vocab={out_of_vocab_count} "+ f"regex_hits={regex_hits} first_token_ids={token_ids[:24]} text_preview='{text_preview}'")- text = ""- token_ids: list[int] = []- if outputs:- text, token_ids = self._extract_text_and_tokens(outputs[0])- if (not text) and token_ids:- try:- text = self.tokenizer.decode(token_ids, skip_special_tokens=False)- except Exception:- text = ""-- if callback is not None:- callback(- step_idx=max(0, max_tokens - 1),- override_num_inference_steps=max_tokens,- denoising_extra=f"{progress_label} {max_tokens}/{max_tokens}",- progress_unit="tokens",- )-- if release_vram_after:- self._release_vram()-- return {"token_ids": token_ids, "text": text}--def create_qwen3_lm_engine(- engine_name: str,- model,- tokenizer,- device,- lm_weights_path: str,- audio_code_mask: Optional[torch.Tensor],- audio_code_token_map: dict[int, int],- free_mmap_after_cuda_copy: bool = True,- weight_load_mode: str = "lazy",-):- engine = (engine_name or "legacy").strip().lower()- if engine == "vllm":- if not lm_weights_path or tokenizer is None:- return None- return Qwen3VllmEngine(- lm_weights_path=lm_weights_path,- tokenizer=tokenizer,- audio_code_mask=audio_code_mask,- audio_code_token_map=audio_code_token_map,- free_mmap_after_cuda_copy=free_mmap_after_cuda_copy,- weight_load_mode=weight_load_mode,- )- if engine == "pt":- if model is None or tokenizer is None:- return None- return Qwen3PtEngine(- model=model,- tokenizer=tokenizer,- device=device,- audio_code_mask=audio_code_mask,- audio_code_token_map=audio_code_token_map,- )- return None+ return _postprocess_audio_codes(parsed_codes, min_tokens, max_tokens), failure_reasondiff --git a/models/TTS/ace_step15/qwen3_lm_engines.py b/models/TTS/ace_step15/qwen3_lm_engines.pynew file mode 100644index 0000000..8d98c3a--- /dev/null+++ b/models/TTS/ace_step15/qwen3_lm_engines.py@@ -0,0 +1,158 @@+"""Qwen3 LM engine adapters."""++from __future__ import annotations++from typing import Any, Callable, Optional, Protocol, runtime_checkable++from shared.llm_engines.nanovllm.vllm_support import NanoVllmTextEngine++from .qwen3_audio_codes import generate_text_sampling+++@runtime_checkable+class Qwen3LmEngine(Protocol):+ keep_loaded_for_phase2: bool++ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):+ ...++ def release_runtime_allocations(self):+ ...++ def get_last_failure_reason(self) -> str:+ ...++ def generate_text(+ self,+ prompt: str,+ prompt_negative: str,+ max_tokens: int,+ temperature: Optional[float],+ top_p: Optional[float],+ top_k: Optional[int],+ cfg_scale: float,+ seed: Optional[int],+ callback=None,+ abort_fn: Optional[Callable[[], bool]] = None,+ logits_processor: Optional[Callable[[Any, Any], Any]] = None,+ logits_processor_update_state: Optional[Callable[[int], None]] = None,+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,+ progress_label: str = "LM text",+ release_vram_after: bool = True,+ ignore_eos: bool = False,+ ):+ ...+++class Qwen3LegacyEngine:+ keep_loaded_for_phase2 = False++ def __init__(self, model, tokenizer, device):+ self.model = model+ self.tokenizer = tokenizer+ self.device = device++ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):+ del prompt_len, max_tokens, cfg_scale++ def release_runtime_allocations(self):+ return++ def get_last_failure_reason(self) -> str:+ return ""++ def generate_text(+ self,+ prompt: str,+ prompt_negative: str,+ max_tokens: int,+ temperature: Optional[float],+ top_p: Optional[float],+ top_k: Optional[int],+ cfg_scale: float,+ seed: Optional[int],+ callback=None,+ abort_fn: Optional[Callable[[], bool]] = None,+ logits_processor: Optional[Callable[[Any, Any], Any]] = None,+ logits_processor_update_state: Optional[Callable[[int], None]] = None,+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,+ progress_label: str = "LM text",+ release_vram_after: bool = True,+ ignore_eos: bool = False,+ ):+ del release_vram_after+ return generate_text_sampling(+ model=self.model,+ tokenizer=self.tokenizer,+ device=self.device,+ prompt=prompt,+ prompt_negative=prompt_negative,+ max_tokens=max_tokens,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ cfg_scale=cfg_scale,+ seed=seed,+ callback=callback,+ abort_fn=abort_fn,+ logits_processor=logits_processor,+ logits_processor_update_state=logits_processor_update_state,+ stop_checker=stop_checker,+ progress_label=progress_label,+ ignore_eos=ignore_eos,+ )+++class Qwen3VllmEngine(NanoVllmTextEngine):+ keep_loaded_for_phase2 = True++ def __init__(self, model, tokenizer, device, lm_weights_path: str):+ del device+ super().__init__(model=model, model_path=lm_weights_path, tokenizer=tokenizer)++ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):+ return super().reserve_runtime(prompt_len=prompt_len, max_tokens=max_tokens, cfg_scale=cfg_scale)++ def release_runtime_allocations(self):+ return super().release_runtime_allocations()++ def get_last_failure_reason(self) -> str:+ return super().get_last_failure_reason()++ def generate_text(+ self,+ prompt: str,+ prompt_negative: str,+ max_tokens: int,+ temperature: Optional[float],+ top_p: Optional[float],+ top_k: Optional[int],+ cfg_scale: float,+ seed: Optional[int],+ callback=None,+ abort_fn: Optional[Callable[[], bool]] = None,+ logits_processor: Optional[Callable[[Any, Any], Any]] = None,+ logits_processor_update_state: Optional[Callable[[int], None]] = None,+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,+ progress_label: str = "LM text",+ release_vram_after: bool = True,+ ignore_eos: bool = False,+ ):+ return super().generate_text(+ prompt=prompt,+ prompt_negative=prompt_negative,+ max_tokens=max_tokens,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ cfg_scale=cfg_scale,+ seed=seed,+ callback=callback,+ abort_fn=abort_fn,+ logits_processor=logits_processor,+ logits_processor_update_state=logits_processor_update_state,+ stop_checker=stop_checker,+ progress_label=progress_label,+ release_vram_after=release_vram_after,+ ignore_eos=ignore_eos,+ )diff --git a/models/TTS/ace_step_handler.py b/models/TTS/ace_step_handler.pyindex 57905cc..7557469 100644--- a/models/TTS/ace_step_handler.py+++ b/models/TTS/ace_step_handler.py@@ -490,7 +490,6 @@ class family_handler:lm_weights = text_encoder_filenamelm_tokenizer_dir = _get_model_path(model_def, "ace_step15_lm_tokenizer_dir", _ace_step15_lm_ckpt_dir(lm_folder))silence_latent = _get_model_path(model_def, "ace_step15_silence_latent", _ace_step15_ckpt_file(ACE_STEP15_SILENCE_LATENT_NAME))- lm_vllm_weight_mode = _get_model_path(model_def, "ace_step15_vllm_weight_mode", "lazy")if enable_lm:lm_weight_name = os.path.basename(str(lm_weights)) if lm_weights else ""print(f"[ace_step15] LM engine='{lm_decoder_engine}' | LM weights='{lm_weight_name}'")@@ -508,7 +507,6 @@ class family_handler:enable_lm=enable_lm,ignore_lm_cache_seed=ignore_lm_cache_seed,lm_decoder_engine=lm_decoder_engine,- lm_vllm_weight_mode=lm_vllm_weight_mode,dtype=dtype or torch.bfloat16,)@@ -517,9 +515,11 @@ class family_handler:"text_encoder_2": pipeline.text_encoder_2,"codec": pipeline.audio_vae,}- if lm_decoder_engine != "vllm" and text_encoder_filename and pipeline.lm_model is not None:+ if text_encoder_filename and pipeline.lm_model is not None:pipe["text_encoder"] = pipeline.lm_model+ pipe = { "pipe": pipe, "coTenantsMap": {}, }+if save_quantized and transformer_weights:from wgp import save_quantized_modeldiff --git a/shared/llm_engines/nanovllm/config.py b/shared/llm_engines/nanovllm/config.pyindex c39c3ea..55329ec 100644--- a/shared/llm_engines/nanovllm/config.py+++ b/shared/llm_engines/nanovllm/config.py@@ -12,9 +12,7 @@ class Config:gpu_memory_utilization: float = 0.9tensor_parallel_size: int = 1enforce_eager: bool = False- weight_load_mode: str = "lazy" # lazy | pinnedhf_config: AutoConfig | None = None- free_mmap_after_cuda_copy: bool = Trueeos: int = -1kvcache_block_size: int = 256num_kvcache_blocks: int = -1@@ -30,8 +28,6 @@ class Config:self.model_dir = self.modelassert self.kvcache_block_size % 256 == 0assert 1 <= self.tensor_parallel_size <= 8- if self.weight_load_mode not in ("lazy", "pinned"):- raise ValueError(f"Unsupported weight_load_mode '{self.weight_load_mode}'. Supported: lazy, pinned.")self.hf_config = AutoConfig.from_pretrained(self.model_dir)self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings)assert self.max_num_batched_tokens >= self.max_model_lendiff --git a/shared/llm_engines/nanovllm/engine/block_manager.py b/shared/llm_engines/nanovllm/engine/block_manager.pyindex 4954044..640b541 100644--- a/shared/llm_engines/nanovllm/engine/block_manager.py+++ b/shared/llm_engines/nanovllm/engine/block_manager.py@@ -2,7 +2,7 @@ from collections import dequeimport xxhashimport numpy as np-from nanovllm.engine.sequence import Sequence+from .sequence import Sequenceclass Block:diff --git a/shared/llm_engines/nanovllm/engine/llm_engine.py b/shared/llm_engines/nanovllm/engine/llm_engine.pyindex 6edec65..8c42fdf 100644--- a/shared/llm_engines/nanovllm/engine/llm_engine.py+++ b/shared/llm_engines/nanovllm/engine/llm_engine.py@@ -5,17 +5,18 @@ from tqdm.auto import tqdmfrom transformers import AutoTokenizerimport torch.multiprocessing as mp-from nanovllm.config import Config-from nanovllm.sampling_params import SamplingParams-from nanovllm.engine.sequence import Sequence-from nanovllm.engine.scheduler import Scheduler-from nanovllm.engine.block_manager import BlockManager-from nanovllm.engine.model_runner import ModelRunner+from ..config import Config+from ..sampling_params import SamplingParams+from .sequence import Sequence+from .scheduler import Scheduler+from .block_manager import BlockManager+from .model_runner import ModelRunnerclass LLMEngine:def __init__(self, model, **kwargs):+ model_object = kwargs.get("model_object", None)config_fields = {field.name for field in fields(Config)}config_kwargs = {k: v for k, v in kwargs.items() if k in config_fields}config = Config(model, **config_kwargs)@@ -29,7 +30,7 @@ class LLMEngine:process.start()self.ps.append(process)self.events.append(event)- self.model_runner = ModelRunner(config, 0, self.events)+ self.model_runner = ModelRunner(config, 0, self.events, model_object=model_object)tokenizer = kwargs.get("tokenizer", None)if tokenizer is not None:self.tokenizer = tokenizer@@ -71,7 +72,7 @@ class LLMEngine:returnself._closed = Truetry:- self.unload_weights()+ self.reset_runtime_state()except Exception:passtry:@@ -98,12 +99,12 @@ class LLMEngine:except Exception:pass- def unload_weights(self):+ def reset_runtime_state(self):runner = getattr(self, "model_runner", None)if runner is None:return- runner.unload_weights()- # KV cache is invalid after unload/reload, so cached prefix block metadata+ runner.reset_runtime_state()+ # KV cache is invalid after runtime reset/reprepare, so cached prefix block metadata# must be dropped as well to prevent stale-cache reuse.try:self.reset()@@ -124,18 +125,6 @@ class LLMEngine:returnrunner.clear_graph_cache()- def reset_guard_counts(self):- runner = getattr(self, "model_runner", None)- if runner is None:- return- runner.call("reset_guard_counts")-- def get_guard_counts(self, reset: bool = False):- runner = getattr(self, "model_runner", None)- if runner is None:- return {}- return runner.call("get_guard_counts", reset)-def add_request(self, prompt: str | list[int], sampling_params: SamplingParams, unconditional_prompt: str | list[int] | None = None):if isinstance(prompt, str):prompt = self.tokenizer.encode(prompt)@@ -208,8 +197,8 @@ class LLMEngine:) -> list[str]:if self.scheduler is None:raise RuntimeError("LLM engine is closed.")- # Ensure weights/KV cache are ready for lazy/pinned modes, and sync scheduler blocks.- self.model_runner.ensure_weights_loaded()+ # Ensure model runtime/KV cache are prepared, and sync scheduler blocks.+ self.model_runner.ensure_runtime_ready()if (self.config.num_kvcache_blocks > 0 andlen(self.scheduler.block_manager.blocks) != self.config.num_kvcache_blocks):self.scheduler.block_manager = BlockManager(diff --git a/shared/llm_engines/nanovllm/engine/model_runner.py b/shared/llm_engines/nanovllm/engine/model_runner.pyindex 9b765ee..aa1e12a 100644--- a/shared/llm_engines/nanovllm/engine/model_runner.py+++ b/shared/llm_engines/nanovllm/engine/model_runner.py@@ -6,12 +6,10 @@ from multiprocessing.synchronize import Eventfrom multiprocessing.shared_memory import SharedMemoryimport sys-from nanovllm.config import Config-from nanovllm.engine.sequence import Sequence-from nanovllm.models import resolve_model_class-from nanovllm.layers.sampler import Sampler-from nanovllm.utils.context import set_context, get_context, reset_context-from nanovllm.utils.loader import load_model, WeightStore+from ..config import Config+from .sequence import Sequence+from ..layers.sampler import Sampler+from ..utils.context import set_context, get_context, reset_contextimport socket@@ -44,7 +42,7 @@ def find_available_port(start_port: int = 2333, max_attempts: int = 100) -> int:class ModelRunner:- def __init__(self, config: Config, rank: int, event: Event | list[Event]):+ def __init__(self, config: Config, rank: int, event: Event | list[Event], model_object=None):# Enable capturing scalar outputs to avoid graph breaks from Tensor.item() callstorch._dynamo.config.capture_scalar_outputs = True@@ -91,12 +89,7 @@ class ModelRunner:config_dtype = torch.bfloat16self.dtype = config_dtype # Save for later use- self.weight_load_mode = (config.weight_load_mode or "lazy").lower()- self._weights_loaded = False- self._weight_store = None- self._is_quanto_int8 = False- self._model_cls = resolve_model_class(hf_config)- self._model_is_meta = False+ self._runtime_ready = Falseself._graph_cache = {}self._graph_cache_order = []self._logits_bias_cache = {}@@ -104,7 +97,12 @@ class ModelRunner:self._guard_counts = {}self._guard_seen_details = set()torch.set_default_dtype(config_dtype)- self.model = self._build_deferred_model(hf_config)+ if model_object is None:+ raise RuntimeError(+ "nanovllm now requires a preloaded MMGP model object. "+ "Pass model_object=... when creating LLM."+ )+ self.model = model_objectself.sampler = Sampler()# Pre-allocate buffers for sampling (optimization: avoid repeated tensor creation)@@ -123,78 +121,16 @@ class ModelRunner:self.shm = SharedMemory(name="nanovllm")self.loop()- def _build_deferred_model(self, hf_config):- # Lazy/pinned mode should not allocate full CPU parameter storage upfront.- with torch.device("meta"):- model = self._model_cls(hf_config)- self._model_is_meta = True- return model-- def ensure_weights_loaded(self):- if self._weights_loaded:- return- print(f"Loading LM Weights '{self.config.model}' in vllm...")- default_dtype = torch.get_default_dtype()- torch.set_default_dtype(self.dtype)- try:- self._ensure_weight_store()- if self.model is None:- self.model = self._build_deferred_model(self.config.hf_config)- self._tie_word_embeddings_if_needed()- self._mark_tied_meta_parameter_ids()- # Tie before materialization so lm_head does not allocate a duplicate tensor.- was_meta_model = bool(self._model_is_meta)- if self._is_quanto_int8:- for module in self.model.modules():- prepare = getattr(module, "prepare_for_quantized_load", None)- if callable(prepare):- prepare()- # Keep model meta until each parameter is replaced directly from safetensor- # to avoid allocating a dense, fully materialized CUDA shell first.- load_model(- self.model,- "",- weight_store=self._weight_store,- # We keep tensors tied to the active weight reader only within each- # load iteration; explicit cloning is not needed and avoids a full- # model-sized temporary copy in RAM.- clone_loaded_tensors=False,- )- if was_meta_model:- self._materialize_runtime_buffers()- self._model_is_meta = False- self._close_weight_store_if_lazy()- self._weights_loaded = True- self.allocate_kv_cache()- if not self.enforce_eager:- self.capture_cudagraph()- finally:- if not self._weights_loaded:- self._close_weight_store_if_lazy()- torch.set_default_device("cpu")- torch.set_default_dtype(default_dtype)-- def _ensure_weight_store(self):- if self._weight_store is not None:- return- self._weight_store = WeightStore(self.config.model_file or self.config.model_dir, mode=self.weight_load_mode)- self._is_quanto_int8 = bool(getattr(self._weight_store, "is_quanto_int8", False))-- def _close_weight_store_if_lazy(self):- if self._weight_store is None:+ def ensure_runtime_ready(self):+ if self._runtime_ready:return- close_fn = getattr(self._weight_store, "close", None)- if callable(close_fn):- close_fn()- self._weight_store = None-- def _materialize_runtime_buffers(self):if self.model is None:- return- for module in self.model.modules():- materialize = getattr(module, "materialize_cache", None)- if callable(materialize):- materialize()+ raise RuntimeError("LLM model object is not available.")+ self._tie_word_embeddings_if_needed()+ self.allocate_kv_cache()+ if not self.enforce_eager:+ self.capture_cudagraph()+ self._runtime_ready = Truedef _get_tied_embeddings(self):if self.model is None:@@ -212,17 +148,6 @@ class ModelRunner:return Nonereturn lm_head, embed- def _mark_tied_meta_parameter_ids(self):- tied = self._get_tied_embeddings()- if tied is None:- self.model._vllm_tied_param_ids = set()- return- lm_head, embed = tied- if lm_head.weight is not embed.weight:- self.model._vllm_tied_param_ids = set()- return- self.model._vllm_tied_param_ids = {id(embed.weight)}-def _tie_word_embeddings_if_needed(self):tied = self._get_tied_embeddings()if tied is None:@@ -233,8 +158,8 @@ class ModelRunner:if lm_head.weight is not embed.weight:raise RuntimeError("Failed to retie lm_head.weight with embed_tokens.weight.")- def unload_weights(self):- if not self._weights_loaded:+ def reset_runtime_state(self):+ if not self._runtime_ready:return# Clear attention KV cache refs so we don't write into freed storage later.try:@@ -249,8 +174,8 @@ class ModelRunner:del self.kv_cacheexcept Exception:pass- # CUDA graphs captured against previous weight/KV pointers are unsafe after unload/reload.- # Force recapture on next load to avoid stale-pointer illegal memory access.+ # CUDA graphs captured against previous model/KV pointers are unsafe after runtime reset.+ # Force recapture on next prepare to avoid stale-pointer illegal memory access.try:self.clear_graph_cache()except Exception:@@ -268,17 +193,7 @@ class ModelRunner:passself._logits_bias_cache.clear()self._sampling_generator = None- self._weights_loaded = False- self.model = None- self._model_is_meta = False- if self._weight_store is not None:- try:- close_fn = getattr(self._weight_store, "close", None)- if callable(close_fn):- close_fn()- except Exception:- pass- self._weight_store = None+ self._runtime_ready = Falsegc.collect()def _get_graph_capture_signature(self):@@ -336,16 +251,6 @@ class ModelRunner:if count == 1:print(f"[nanovllm][guard] {name}")- def reset_guard_counts(self):- self._guard_counts.clear()- self._guard_seen_details.clear()-- def get_guard_counts(self, reset: bool = False):- counts = dict(self._guard_counts)- if reset:- self.reset_guard_counts()- return counts-def _get_logits_bias(self, seq: Sequence, logits: torch.Tensor):bias = getattr(seq, "logits_bias", None)if bias is None or not torch.is_tensor(bias):@@ -430,17 +335,9 @@ class ModelRunner:def exit(self):try:- self.unload_weights()- except Exception:- pass- try:- if self._weight_store is not None:- close_fn = getattr(self._weight_store, "close", None)- if callable(close_fn):- close_fn()+ self.reset_runtime_state()except Exception:pass- self._weight_store = Noneself._release_sample_buffers()self._logits_bias_cache.clear()self._guard_counts.clear()@@ -449,7 +346,6 @@ class ModelRunner:self.sampler = Noneif hasattr(self, "model"):self.model = None- self._model_is_meta = Falseif self.world_size > 1:self.shm.close()dist.barrier()@@ -710,7 +606,7 @@ class ModelRunner:"""Run model forward and sampling. For CFG sequences, batch is structured as:[cond_seq1, cond_seq2, ..., uncond_seq1, uncond_seq2, ...]where uncond_seqi is the paired unconditional sequence of cond_seqi."""- self.ensure_weights_loaded()+ self.ensure_runtime_ready()# Check if this is a CFG batch (contains paired conditional and unconditional sequences)is_cfg_batch = seqs[0].cfg_scale > 1.0 and seqs[0].paired_seq is not Noneif is_cfg_batch:@@ -878,7 +774,7 @@ class ModelRunner:def capture_cudagraph(self):config = self.configcache_key = (config.max_model_len, config.max_num_seqs)- model_device = self._get_model_device()+ model_device = torch.device("cuda") if torch.cuda.is_available() else self._get_model_device()cached = self._graph_cache.get(cache_key)if cached is not None:current_sig = self._get_graph_capture_signature()diff --git a/shared/llm_engines/nanovllm/engine/scheduler.py b/shared/llm_engines/nanovllm/engine/scheduler.pyindex 3178c22..60162a9 100644--- a/shared/llm_engines/nanovllm/engine/scheduler.py+++ b/shared/llm_engines/nanovllm/engine/scheduler.py@@ -1,8 +1,8 @@from collections import deque-from nanovllm.config import Config-from nanovllm.engine.sequence import Sequence, SequenceStatus-from nanovllm.engine.block_manager import BlockManager+from ..config import Config+from .sequence import Sequence, SequenceStatus+from .block_manager import BlockManagerclass Scheduler:diff --git a/shared/llm_engines/nanovllm/engine/sequence.py b/shared/llm_engines/nanovllm/engine/sequence.pyindex bda2e53..86869d5 100644--- a/shared/llm_engines/nanovllm/engine/sequence.py+++ b/shared/llm_engines/nanovllm/engine/sequence.py@@ -3,7 +3,7 @@ from enum import Enum, autofrom itertools import countfrom typing import Optional, Callable, Any-from nanovllm.sampling_params import SamplingParams+from ..sampling_params import SamplingParamsclass SequenceStatus(Enum):diff --git a/shared/llm_engines/nanovllm/layers/attention.py b/shared/llm_engines/nanovllm/layers/attention.pyindex e416139..e03f43c 100644--- a/shared/llm_engines/nanovllm/layers/attention.py+++ b/shared/llm_engines/nanovllm/layers/attention.py@@ -4,7 +4,7 @@ import tritonimport triton.language as tlfrom flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache-from nanovllm.utils.context import get_context+from ..utils.context import get_context@triton.jitdiff --git a/shared/llm_engines/nanovllm/layers/embed_head.py b/shared/llm_engines/nanovllm/layers/embed_head.pyindex 4475b41..793063a 100644--- a/shared/llm_engines/nanovllm/layers/embed_head.py+++ b/shared/llm_engines/nanovllm/layers/embed_head.py@@ -3,7 +3,7 @@ from torch import nnimport torch.nn.functional as Fimport torch.distributed as dist-from nanovllm.utils.context import get_context+from ..utils.context import get_contextdef _get_tp_info():diff --git a/shared/llm_engines/nanovllm/layers/linear.py b/shared/llm_engines/nanovllm/layers/linear.pyindex ab19766..b503342 100644--- a/shared/llm_engines/nanovllm/layers/linear.py+++ b/shared/llm_engines/nanovllm/layers/linear.py@@ -2,27 +2,6 @@ import torchfrom torch import nnimport torch.nn.functional as Fimport torch.distributed as dist-import math--try:- from shared.kernels import quanto_int8_triton as _shared_quanto_int8_triton-except Exception: # pragma: no cover- _shared_quanto_int8_triton = None # type: ignore---def _shared_kernel_available() -> bool:- if _shared_quanto_int8_triton is None:- return False- is_available = getattr(_shared_quanto_int8_triton, "is_available", None)- if not callable(is_available):- return False- try:- return bool(is_available())- except Exception:- return False---_TRITON_AVAILABLE = _shared_kernel_available()def divide(numerator, denominator):@@ -36,232 +15,15 @@ def _get_tp_info():return 0, 1-def _flatten_scale(scale: torch.Tensor) -> torch.Tensor:- if scale.ndim == 2 and scale.shape[1] == 1:- return scale.view(-1)- if scale.ndim == 1:- return scale- return scale.reshape(-1)---def _run_triton_fused_int8_mm(- x2d: torch.Tensor,- qweight_t: torch.Tensor,- qweight_scale_fp32: torch.Tensor,- input_scale: float | None = None,-) -> torch.Tensor:- del input_scale- if _shared_quanto_int8_triton is None:- raise RuntimeError("shared.kernels.quanto_int8_triton is unavailable")- run_kernel = getattr(_shared_quanto_int8_triton, "fused_quant_scaled_mm_transposed", None)- if run_kernel is None:- raise RuntimeError("shared.kernels.quanto_int8_triton.fused_quant_scaled_mm_transposed is missing")-- m, k = x2d.shape- k2, n = qweight_t.shape- if k != k2:- raise RuntimeError(f"Triton int8 GEMM shape mismatch: x={x2d.shape}, w_t={qweight_t.shape}")- qweight_scale_fp32 = _flatten_scale(qweight_scale_fp32)- if qweight_scale_fp32.numel() != n:- raise RuntimeError(- f"Triton int8 qweight_scale length mismatch: expected {n}, got {qweight_scale_fp32.numel()}"- )- return run_kernel(x2d, qweight_t, qweight_scale_fp32, out_dtype=x2d.dtype)-+class LinearBase(nn.Linear):-class LinearBase(nn.Module):-- def __init__(- self,- input_size: int,- output_size: int,- bias: bool = False,- tp_dim: int | None = None,- ):- super().__init__()- self.tp_dim = tp_dim+ def __init__(self, input_size: int, output_size: int, bias: bool = False):+ super().__init__(input_size, output_size, bias=bias)self.tp_rank, self.tp_size = _get_tp_info()- self.input_size = input_size- self.output_size = output_size- self.weight = nn.Parameter(torch.empty(output_size, input_size))- self.weight.weight_loader = self.weight_loader- self.register_buffer("qweight_data", torch.empty(0, dtype=torch.int8))- self.register_buffer("qweight_t", torch.empty(0, dtype=torch.int8))- self.register_buffer("qweight_scale", torch.empty(0))- self.register_buffer("qweight_scale_fp32", torch.empty(0, dtype=torch.float32))- self.register_buffer("input_scale", torch.ones((), dtype=torch.bfloat16))- self.register_buffer("output_scale", torch.ones((), dtype=torch.bfloat16))- self.use_int8_weight = False- self.use_triton_int8 = False- self._input_scale_value = 1.0- self._quant_expected_shards = 1- self._quant_data_loaded = set()- self._quant_scale_loaded = set()- if bias:- self.bias = nn.Parameter(torch.empty(output_size))- self.bias.weight_loader = self.weight_loader- else:- self.register_parameter("bias", None)def forward(self, x: torch.Tensor) -> torch.Tensor:raise NotImplementedError- def _shard_weight(self, loaded_weight: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:- return loaded_weight-- def _shard_weight_scale(self, loaded_scale: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:- return loaded_scale-- def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id=None):- param.data.copy_(self._shard_weight(loaded_weight, loaded_shard_id))-- def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):- device = self.weight.device- shard = self._shard_weight(loaded_weight, loaded_shard_id)- if self.qweight_t.numel() == 0:- self.qweight_t = torch.empty(- (self.input_size, self.output_size),- dtype=torch.int8,- device=device,- )- if shard.dtype != torch.int8:- shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)- else:- shard = shard.to(device=device, non_blocking=True)- # Store directly in transposed layout to avoid an extra full-size transpose pass- # after loading.- self.qweight_t.t().copy_(shard)- shard_key = loaded_shard_id if loaded_shard_id is not None else 0- self._quant_data_loaded.add(shard_key)-- def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device- shard_scale = self._shard_weight_scale(loaded_scale, loaded_shard_id).contiguous()- if shard_scale.dim() == 2 and shard_scale.size(1) == 1:- shard_scale = shard_scale.squeeze(1)- self.qweight_scale = shard_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)- shard_key = loaded_shard_id if loaded_shard_id is not None else 0- self._quant_scale_loaded.add(shard_key)-- def quant_input_scale_loader(self, loaded_scale: torch.Tensor):- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device- self.input_scale = loaded_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)-- def quant_output_scale_loader(self, loaded_scale: torch.Tensor):- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device- self.output_scale = loaded_scale.to(device=device, dtype=torch.bfloat16, non_blocking=True)-- def finalize_quantized(self):- if self.qweight_scale.numel() == 0:- return- if len(self._quant_data_loaded) < self._quant_expected_shards:- return- if len(self._quant_scale_loaded) < self._quant_expected_shards:- return- if self.qweight_data.numel() == 0 and self.qweight_t.numel() == 0:- return- if self.qweight_t.numel() == 0:- # Fallback path for loaders that still filled qweight_data (older checkpoints- # or custom loaders). Keep memory usage low by releasing qweight_data immediately- # after conversion.- self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()- self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)- self.qweight_scale_fp32 = self.qweight_scale.to(dtype=torch.float32)- if not torch.is_tensor(self.input_scale):- raise RuntimeError(f"Invalid input_scale type: {type(self.input_scale)}")- input_scale_flat = self.input_scale.reshape(-1)- if input_scale_flat.numel() != 1:- raise RuntimeError(- f"Expected scalar input_scale, got shape={tuple(self.input_scale.shape)}"- )- input_scale_value = float(input_scale_flat[0].item())- if not math.isfinite(input_scale_value) or input_scale_value <= 0:- raise RuntimeError(f"Invalid input_scale value: {input_scale_value}")- self._input_scale_value = max(input_scale_value, 1e-8)- self.use_triton_int8 = bool(_TRITON_AVAILABLE and self.qweight_t.is_cuda)- self.use_int8_weight = True- device = self.qweight_t.device- if self.weight.numel() != 0:- self.weight = nn.Parameter(torch.empty(0, device=device, dtype=torch.bfloat16), requires_grad=False)- if self.bias is not None:- self.bias = nn.Parameter(self.bias.data.to(device=device), requires_grad=False)-- def _quant_int8_mm(self, x: torch.Tensor) -> torch.Tensor:- x_shape = x.shape- x2d = x.reshape(-1, x_shape[-1])- if self.use_triton_int8 and x2d.is_cuda and self.qweight_t.numel() != 0 and self.qweight_scale_fp32.numel() != 0:- y = _run_triton_fused_int8_mm(- x2d,- self.qweight_t,- self.qweight_scale_fp32,- )- return y.view(*x_shape[:-1], y.size(-1))-- # Fallback path: use Quanto qbytes_mm for dynamic activation quantization parity.- if not hasattr(torch.ops, "quanto") or not hasattr(torch.ops.quanto, "qbytes_mm"):- raise RuntimeError("quanto.qbytes_mm op unavailable for int8 fallback path")- if self.qweight_t.numel() == 0 and self.qweight_data.numel() != 0:- self.qweight_t = self.qweight_data.transpose(0, 1).contiguous()- self.qweight_data = torch.empty(0, dtype=torch.int8, device=self.qweight_t.device)- qweight = self.qweight_t.transpose(0, 1).contiguous()- scales = self.qweight_scale_fp32- if scales.numel() == 0:- scales = self.qweight_scale.to(torch.float32)- scales = _flatten_scale(scales).reshape(-1, 1).contiguous()- out = torch.ops.quanto.qbytes_mm(x2d, qweight, scales).to(x2d.dtype)- return out.view(*x_shape[:-1], out.size(-1))-- def prepare_for_quantized_load(self):- device = None- if self.qweight_t.numel() != 0:- device = self.qweight_t.device- elif self.qweight_data.numel() != 0:- device = self.qweight_data.device- elif self.weight.numel() != 0:- device = self.weight.device- elif self.qweight_scale.numel() != 0:- device = self.qweight_scale.device- else:- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")- if getattr(device, "type", None) == "meta":- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")-- if self.weight.numel() != 0:- self.weight = nn.Parameter(torch.empty(0, device=device, dtype=torch.bfloat16), requires_grad=False)-- # Reset all quantized buffers/state so each reload is complete and deterministic.- self.qweight_data = torch.empty(0, dtype=torch.int8, device=device)- self.qweight_t = torch.empty(0, dtype=torch.int8, device=device)- self.qweight_scale = torch.empty(0, dtype=torch.bfloat16, device=device)- self.qweight_scale_fp32 = torch.empty(0, dtype=torch.float32, device=device)- self.input_scale = torch.ones((), dtype=torch.bfloat16, device=device)- self.output_scale = torch.ones((), dtype=torch.bfloat16, device=device)- self.use_int8_weight = False- self.use_triton_int8 = False- self._input_scale_value = 1.0- self._quant_data_loaded.clear()- self._quant_scale_loaded.clear()---class ReplicatedLinear(LinearBase):-- def __init__(- self,- input_size: int,- output_size: int,- bias: bool = False,- ):- super().__init__(input_size, output_size, bias)-- def forward(self, x: torch.Tensor) -> torch.Tensor:- if self.use_int8_weight:- y = self._quant_int8_mm(x)- if self.bias is not None:- y = y + self.bias- return y- return F.linear(x, self.weight, self.bias)--class ColumnParallelLinear(LinearBase):def __init__(@@ -271,144 +33,12 @@ class ColumnParallelLinear(LinearBase):bias: bool = False,):tp_size = _get_tp_info()[1]- super().__init__(input_size, divide(output_size, tp_size), bias, 0)-- def _shard_weight(self, loaded_weight: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:- shard_size = divide(loaded_weight.size(self.tp_dim), self.tp_size)- start_idx = self.tp_rank * shard_size- return loaded_weight.narrow(self.tp_dim, start_idx, shard_size)-- def _shard_weight_scale(self, loaded_scale: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:- if loaded_scale.dim() == 0:- return loaded_scale- shard_size = divide(loaded_scale.size(0), self.tp_size)- start_idx = self.tp_rank * shard_size- return loaded_scale.narrow(0, start_idx, shard_size)+ super().__init__(input_size, divide(output_size, tp_size), bias)def forward(self, x: torch.Tensor) -> torch.Tensor:- if self.use_int8_weight:- y = self._quant_int8_mm(x)- if self.bias is not None:- y = y + self.bias- return yreturn F.linear(x, self.weight, self.bias)-class MergedColumnParallelLinear(ColumnParallelLinear):-- def __init__(- self,- input_size: int,- output_sizes: list[int],- bias: bool = False,- ):- self.output_sizes = output_sizes- super().__init__(input_size, sum(output_sizes), bias)- self._quant_expected_shards = len(output_sizes)-- def _merged_shard_meta(self, loaded_shard_id: int):- shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size- shard_size = self.output_sizes[loaded_shard_id] // self.tp_size- return shard_offset, shard_size-- def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int):- param_data = param.data- shard_offset, shard_size = self._merged_shard_meta(loaded_shard_id)- param_slice = param_data.narrow(self.tp_dim, shard_offset, shard_size)- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]- param_slice.copy_(shard)-- def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):- device = self.weight.device- if self.qweight_t.numel() == 0:- self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)- shard_offset, shard_size = self._merged_shard_meta(int(loaded_shard_id))- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]- if shard.dtype != torch.int8:- shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)- else:- shard = shard.to(device=device, non_blocking=True)- self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())- self._quant_data_loaded.add(int(loaded_shard_id))-- def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device- if self.qweight_scale.numel() == 0:- self.qweight_scale = torch.empty((self.output_size,), dtype=torch.bfloat16, device=device)- shard_offset, shard_size = self._merged_shard_meta(int(loaded_shard_id))- scale_slice = self.qweight_scale.narrow(0, shard_offset, shard_size)- shard = loaded_scale.chunk(self.tp_size, 0)[self.tp_rank]- if shard.dim() == 2 and shard.size(1) == 1:- shard = shard.squeeze(1)- scale_slice.copy_(shard.to(device=device, dtype=torch.bfloat16, non_blocking=True))- self._quant_scale_loaded.add(int(loaded_shard_id))---class QKVParallelLinear(ColumnParallelLinear):-- def __init__(- self,- hidden_size: int,- head_size: int,- total_num_heads: int,- total_num_kv_heads: int | None = None,- bias: bool = False,- ):- tp_size = _get_tp_info()[1]- total_num_kv_heads = total_num_kv_heads or total_num_heads- self.head_size = head_size- self.num_heads = divide(total_num_heads, tp_size)- self.num_kv_heads = divide(total_num_kv_heads, tp_size)- output_size = (total_num_heads + 2 * total_num_kv_heads) * self.head_size- super().__init__(hidden_size, output_size, bias)- self._quant_expected_shards = 3-- def _qkv_offset_size(self, loaded_shard_id: str):- assert loaded_shard_id in ["q", "k", "v"]- if loaded_shard_id == "q":- shard_size = self.num_heads * self.head_size- shard_offset = 0- elif loaded_shard_id == "k":- shard_size = self.num_kv_heads * self.head_size- shard_offset = self.num_heads * self.head_size- else:- shard_size = self.num_kv_heads * self.head_size- shard_offset = self.num_heads * self.head_size + self.num_kv_heads * self.head_size- return shard_offset, shard_size-- def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str):- param_data = param.data- shard_offset, shard_size = self._qkv_offset_size(loaded_shard_id)- param_slice = param_data.narrow(self.tp_dim, shard_offset, shard_size)- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]- param_slice.copy_(shard)-- def quant_weight_data_loader(self, loaded_weight: torch.Tensor, loaded_shard_id=None):- device = self.weight.device- if self.qweight_t.numel() == 0:- self.qweight_t = torch.empty((self.input_size, self.output_size), dtype=torch.int8, device=device)- shard_offset, shard_size = self._qkv_offset_size(str(loaded_shard_id))- shard = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank]- if shard.dtype != torch.int8:- shard = shard.to(device=device, dtype=torch.int8, non_blocking=True)- else:- shard = shard.to(device=device, non_blocking=True)- self.qweight_t.narrow(1, shard_offset, shard_size).copy_(shard.t())- self._quant_data_loaded.add(str(loaded_shard_id))-- def quant_weight_scale_loader(self, loaded_scale: torch.Tensor, loaded_shard_id=None):- device = self.weight.device if self.weight.numel() != 0 else self.qweight_data.device- if self.qweight_scale.numel() == 0:- self.qweight_scale = torch.empty((self.output_size,), dtype=torch.bfloat16, device=device)- shard_offset, shard_size = self._qkv_offset_size(str(loaded_shard_id))- scale_slice = self.qweight_scale.narrow(0, shard_offset, shard_size)- shard = loaded_scale.chunk(self.tp_size, 0)[self.tp_rank]- if shard.dim() == 2 and shard.size(1) == 1:- shard = shard.squeeze(1)- scale_slice.copy_(shard.to(device=device, dtype=torch.bfloat16, non_blocking=True))- self._quant_scale_loaded.add(str(loaded_shard_id))--class RowParallelLinear(LinearBase):def __init__(@@ -418,24 +48,10 @@ class RowParallelLinear(LinearBase):bias: bool = False,):tp_size = _get_tp_info()[1]- super().__init__(divide(input_size, tp_size), output_size, bias, 1)-- def _shard_weight(self, loaded_weight: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:- shard_size = divide(loaded_weight.size(self.tp_dim), self.tp_size)- start_idx = self.tp_rank * shard_size- return loaded_weight.narrow(self.tp_dim, start_idx, shard_size)-- def _shard_weight_scale(self, loaded_scale: torch.Tensor, loaded_shard_id=None) -> torch.Tensor:- # Output rows are not sharded in RowParallelLinear.- return loaded_scale+ super().__init__(divide(input_size, tp_size), output_size, bias)def forward(self, x: torch.Tensor) -> torch.Tensor:- if self.use_int8_weight:- y = self._quant_int8_mm(x)- if self.bias is not None and self.tp_rank == 0:- y = y + self.bias- else:- y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None)+ y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None)if self.tp_size > 1:dist.all_reduce(y)return ydiff --git a/shared/llm_engines/nanovllm/layers/rotary_embedding.py b/shared/llm_engines/nanovllm/layers/rotary_embedding.pyindex acaeffd..3227623 100644--- a/shared/llm_engines/nanovllm/layers/rotary_embedding.py+++ b/shared/llm_engines/nanovllm/layers/rotary_embedding.py@@ -13,6 +13,16 @@ def apply_rotary_emb(y2 = x2 * cos + x1 * sinreturn torch.cat((y1, y2), dim=-1).to(x.dtype)+@lru_cache(1)+def _build_cache(device: torch.device | str | None, base, rotary_dim, max_position_embeddings ) -> torch.Tensor:+ inv_freq = 1.0 / (+ base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device) / rotary_dim)+ )+ t = torch.arange(max_position_embeddings, dtype=torch.float32, device=device)+ freqs = torch.einsum("i,j -> ij", t, inv_freq)+ cos = freqs.cos()+ sin = freqs.sin()+ return torch.cat((cos, sin), dim=-1).unsqueeze_(1)class RotaryEmbedding(nn.Module):@@ -27,28 +37,12 @@ class RotaryEmbedding(nn.Module):self.head_size = head_sizeself.rotary_dim = rotary_dimself.max_position_embeddings = max_position_embeddings- self.base = float(base)+ base = float(base)+ self.base = baseassert rotary_dim == head_size- cache = self._build_cache(device=None)+ cache = _build_cache("cpu", base, rotary_dim, max_position_embeddings )self.register_buffer("cos_sin_cache", cache, persistent=False)- def _build_cache(self, device: torch.device | str | None) -> torch.Tensor:- inv_freq = 1.0 / (- self.base ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32, device=device) / self.rotary_dim)- )- t = torch.arange(self.max_position_embeddings, dtype=torch.float32, device=device)- freqs = torch.einsum("i,j -> ij", t, inv_freq)- cos = freqs.cos()- sin = freqs.sin()- return torch.cat((cos, sin), dim=-1).unsqueeze_(1)-- def materialize_cache(self, device: torch.device | str | None = None):- target_device = device- if target_device is None:- target_device = self.cos_sin_cache.device- if getattr(target_device, "type", None) == "meta":- target_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")- self.cos_sin_cache = self._build_cache(target_device)@torch.compiledef forward(@@ -63,15 +57,3 @@ class RotaryEmbedding(nn.Module):key = apply_rotary_emb(key, cos, sin)return query, key--@lru_cache(1)-def get_rope(- head_size: int,- rotary_dim: int,- max_position: int,- base: float,- rope_scaling: dict | None = None,-):- assert rope_scaling is None- rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base)- return rotary_embdiff --git a/shared/llm_engines/nanovllm/llm.py b/shared/llm_engines/nanovllm/llm.pyindex 4f51a44..0266054 100644--- a/shared/llm_engines/nanovllm/llm.py+++ b/shared/llm_engines/nanovllm/llm.py@@ -1,4 +1,4 @@-from nanovllm.engine.llm_engine import LLMEngine+from .engine.llm_engine import LLMEngineclass LLM(LLMEngine):diff --git a/shared/llm_engines/nanovllm/models/__init__.py b/shared/llm_engines/nanovllm/models/__init__.pyindex 692e251..67d5754 100644--- a/shared/llm_engines/nanovllm/models/__init__.py+++ b/shared/llm_engines/nanovllm/models/__init__.py@@ -1,52 +1,3 @@-import importlib-from typing import Iterable-from torch import nn+from .qwen3 import Qwen3ForCausalLM--_ARCH_REGISTRY = {- "Qwen3ForCausalLM": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),-}--_MODEL_TYPE_REGISTRY = {- "qwen3": ("nanovllm.models.qwen3", "Qwen3ForCausalLM"),-}---def _load_model_cls(module_name: str, class_name: str):- module = importlib.import_module(module_name)- model_cls = getattr(module, class_name, None)- if model_cls is None:- raise RuntimeError(f"Model class '{class_name}' was not found in '{module_name}'.")- if not isinstance(model_cls, type) or not issubclass(model_cls, nn.Module):- raise RuntimeError(f"Resolved '{module_name}.{class_name}' is not a torch.nn.Module subclass.")- return model_cls---def _iter_architectures(hf_config) -> Iterable[str]:- architectures = getattr(hf_config, "architectures", None)- if isinstance(architectures, (list, tuple)):- for architecture in architectures:- if isinstance(architecture, str) and len(architecture) > 0:- yield architecture---def resolve_model_class(hf_config):- for architecture in _iter_architectures(hf_config):- entry = _ARCH_REGISTRY.get(architecture, None)- if entry is not None:- return _load_model_cls(entry[0], entry[1])-- model_type = str(getattr(hf_config, "model_type", "")).strip().lower()- if model_type:- entry = _MODEL_TYPE_REGISTRY.get(model_type, None)- if entry is not None:- return _load_model_cls(entry[0], entry[1])-- supported_architectures = ", ".join(sorted(_ARCH_REGISTRY.keys()))- supported_model_types = ", ".join(sorted(_MODEL_TYPE_REGISTRY.keys()))- found_architectures = ", ".join(list(_iter_architectures(hf_config))) or "<none>"- raise RuntimeError(- "nanovllm does not support this model architecture. "- f"Found architectures: {found_architectures}; model_type: '{model_type or '<none>'}'. "- f"Supported architectures: {supported_architectures}; supported model_type values: {supported_model_types}."- )+__all__ = ["Qwen3ForCausalLM"]diff --git a/shared/llm_engines/nanovllm/models/qwen3.py b/shared/llm_engines/nanovllm/models/qwen3.pyindex 7b98783..550a358 100644--- a/shared/llm_engines/nanovllm/models/qwen3.py+++ b/shared/llm_engines/nanovllm/models/qwen3.py@@ -3,12 +3,12 @@ from torch import nnimport torch.distributed as distfrom transformers import Qwen3Config-from nanovllm.layers.activation import SiluAndMul-from nanovllm.layers.attention import Attention-from nanovllm.layers.layernorm import RMSNorm-from nanovllm.layers.linear import QKVParallelLinear, MergedColumnParallelLinear, RowParallelLinear-from nanovllm.layers.rotary_embedding import get_rope-from nanovllm.layers.embed_head import VocabParallelEmbedding, ParallelLMHead+from ..layers.activation import SiluAndMul+from ..layers.attention import Attention+from ..layers.layernorm import RMSNorm+from ..layers.linear import ColumnParallelLinear, RowParallelLinear+from ..layers.rotary_embedding import RotaryEmbedding+from ..layers.embed_head import VocabParallelEmbedding, ParallelLMHeaddef _get_tp_size():@@ -45,31 +45,18 @@ class Qwen3Attention(nn.Module):self.scaling = self.head_dim ** -0.5self.qkv_bias = qkv_bias- self.qkv_proj = QKVParallelLinear(- hidden_size,- self.head_dim,- self.total_num_heads,- self.total_num_kv_heads,- bias=qkv_bias,- )+ self.q_proj = ColumnParallelLinear(hidden_size, self.total_num_heads * self.head_dim, bias=qkv_bias)+ self.k_proj = ColumnParallelLinear(hidden_size, self.total_num_kv_heads * self.head_dim, bias=qkv_bias)+ self.v_proj = ColumnParallelLinear(hidden_size, self.total_num_kv_heads * self.head_dim, bias=qkv_bias)self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,hidden_size,bias=False,)- self.rotary_emb = get_rope(- self.head_dim,- rotary_dim=self.head_dim,- max_position=max_position,- base=rope_theta,- rope_scaling=rope_scaling,- )- self.attn = Attention(- self.num_heads,- self.head_dim,- self.scaling,- self.num_kv_heads,- )+ self.rotary_emb = RotaryEmbedding(self.head_dim, self.head_dim, max_position, rope_theta)++ self.attn = Attention( self.num_heads, self.head_dim, self.scaling, self.num_kv_heads)+if not self.qkv_bias:self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)@@ -79,8 +66,9 @@ class Qwen3Attention(nn.Module):positions: torch.Tensor,hidden_states: torch.Tensor,) -> torch.Tensor:- qkv = self.qkv_proj(hidden_states)- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)+ q = self.q_proj(hidden_states)+ k = self.k_proj(hidden_states)+ v = self.v_proj(hidden_states)q = q.view(-1, self.num_heads, self.head_dim)k = k.view(-1, self.num_kv_heads, self.head_dim)v = v.view(-1, self.num_kv_heads, self.head_dim)@@ -102,11 +90,8 @@ class Qwen3MLP(nn.Module):hidden_act: str,) -> None:super().__init__()- self.gate_up_proj = MergedColumnParallelLinear(- hidden_size,- [intermediate_size] * 2,- bias=False,- )+ self.gate_proj = ColumnParallelLinear(hidden_size, intermediate_size, bias=False)+ self.up_proj = ColumnParallelLinear(hidden_size, intermediate_size, bias=False)self.down_proj = RowParallelLinear(intermediate_size,hidden_size,@@ -116,8 +101,9 @@ class Qwen3MLP(nn.Module):self.act_fn = SiluAndMul()def forward(self, x):- gate_up = self.gate_up_proj(x)- x = self.act_fn(gate_up)+ gate = self.gate_proj(x)+ up = self.up_proj(x)+ x = self.act_fn(torch.cat((gate, up), dim=-1))x = self.down_proj(x)return x@@ -189,14 +175,6 @@ class Qwen3Model(nn.Module):class Qwen3ForCausalLM(nn.Module):- packed_modules_mapping = {- "q_proj": ("qkv_proj", "q"),- "k_proj": ("qkv_proj", "k"),- "v_proj": ("qkv_proj", "v"),- "gate_proj": ("gate_up_proj", 0),- "up_proj": ("gate_up_proj", 1),- }-def __init__(self,config: Qwen3Config@@ -205,8 +183,7 @@ class Qwen3ForCausalLM(nn.Module):self.model = Qwen3Model(config)self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)- # Proxy attributes for weight loading compatibility- # Some model weights use "embed_tokens" instead of "model.embed_tokens"+ # Proxy attributes to keep the top-level module layout simple for runtime helpers.@propertydef embed_tokens(self):return self.model.embed_tokensdiff --git a/shared/llm_engines/nanovllm/utils/loader.py b/shared/llm_engines/nanovllm/utils/loader.pydeleted file mode 100644index 0a5b490..0000000--- a/shared/llm_engines/nanovllm/utils/loader.py+++ /dev/null@@ -1,297 +0,0 @@-import gc-import os-from collections import OrderedDict-from glob import glob-import torch-from torch import nn-from safetensors import safe_open---def default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor):- if param.is_meta:- raise RuntimeError("default_weight_loader cannot copy into meta parameter directly.")- if loaded_weight.device != param.device or loaded_weight.dtype != param.dtype:- loaded_weight = loaded_weight.to(- device=param.device,- dtype=param.dtype,- non_blocking=True,- )- param.data.copy_(loaded_weight)---def _coerce_loaded_weight_to_param(param: nn.Parameter, loaded_weight: torch.Tensor) -> torch.Tensor:- target_device = param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")- if loaded_weight.device != target_device:- loaded_weight = loaded_weight.to(device=target_device, non_blocking=True)- if loaded_weight.dtype != param.dtype:- loaded_weight = loaded_weight.to(dtype=param.dtype, non_blocking=True)- return loaded_weight---def _restore_param_attributes(new_param: nn.Parameter, src_param: nn.Parameter) -> None:- for attr_name, attr_value in src_param.__dict__.items():- setattr(new_param, attr_name, attr_value)---def _replace_meta_parameter(- parent: nn.Module,- param_name: str,- loaded_weight: torch.Tensor,- src_param: nn.Parameter,-) -> nn.Parameter:- new_param = nn.Parameter(- loaded_weight,- requires_grad=src_param.requires_grad,- )- parent._parameters[param_name] = new_param- _restore_param_attributes(new_param, src_param)- return new_param---def _replace_tied_meta_parameter(- model: nn.Module,- old_param: nn.Parameter,- loaded_weight: torch.Tensor,-) -> nn.Parameter:- new_param = nn.Parameter(- loaded_weight,- requires_grad=old_param.requires_grad,- )- _restore_param_attributes(new_param, old_param)- for module in model.modules():- for name, param in list(module._parameters.items()):- if param is old_param:- module._parameters[name] = new_param- return new_param---def _should_replace_param_globally(model: nn.Module, param: nn.Parameter) -> bool:- tied_ids = getattr(model, "_vllm_tied_param_ids", None)- return bool(tied_ids) and id(param) in tied_ids---def _candidate_weight_names(weight_name: str):- candidates = [weight_name]- if weight_name.startswith("model."):- candidates.append(weight_name[6:])- else:- candidates.append(f"model.{weight_name}")- return candidates---def _get_parameter_target(model: nn.Module, weight_name: str):- for candidate in _candidate_weight_names(weight_name):- if "." in candidate:- parent_name, param_name = candidate.rsplit(".", 1)- else:- parent_name, param_name = "", candidate- try:- parent = model if parent_name == "" else model.get_submodule(parent_name)- except Exception:- continue- param = parent._parameters.get(param_name)- if param is not None and isinstance(param, nn.Parameter):- return parent, param_name, param- return None---def _get_submodule_safe(model: nn.Module, module_name: str):- for candidate in _candidate_weight_names(module_name):- if candidate == "":- return model- try:- return model.get_submodule(candidate)- except Exception:- pass- return None---def _list_safetensor_files(path: str) -> list[str]:- if os.path.isfile(path) and path.endswith(".safetensors"):- return [path]- return glob(os.path.join(path, "*.safetensors"))---class WeightStore:- def __init__(self, path: str, mode: str = "lazy"):- self.path = path- self.mode = (mode or "lazy").lower()- self.files = _list_safetensor_files(path)- if not self.files:- raise FileNotFoundError(f"No .safetensors files found in {path}")- self._weight_to_file = {}- self._file_to_weight_names = OrderedDict()- self._pinned_weights = {}- self.is_quanto_int8 = False- for file in self.files:- with safe_open(file, "pt", "cpu", writable_tensors=False) as f:- for key in f.keys():- self._weight_to_file[key] = file- self._file_to_weight_names.setdefault(file, []).append(key)- if key.endswith(".weight._data"):- self.is_quanto_int8 = True- if self.mode == "pinned":- self._preload_pinned()-- def _preload_pinned(self):- for file, keys in self._file_to_weight_names.items():- with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:- for key in keys:- tensor = handle.get_tensor(key)- if tensor.device.type != "cpu":- tensor = tensor.cpu()- if not tensor.is_pinned():- tensor = tensor.pin_memory()- self._pinned_weights[key] = tensor-- def close(self):- self._weight_to_file.clear()- self._file_to_weight_names.clear()- self._pinned_weights.clear()-- def __del__(self):- try:- self.close()- except Exception:- pass-- def get_tensor(self, key: str) -> torch.Tensor:- if self.mode == "pinned":- return self._pinned_weights[key]- file = self._weight_to_file.get(key)- if file is None:- raise KeyError(key)- with safe_open(file, "pt", "cpu", writable_tensors=False) as handle:- return handle.get_tensor(key)---def load_model(- model: nn.Module,- path: str,- weight_store: WeightStore | None = None,- clone_loaded_tensors: bool = False,-):- packed_modules_mapping = getattr(model, "packed_modules_mapping", {})- if weight_store is None:- safetensor_files = _list_safetensor_files(path)- if not safetensor_files:- raise FileNotFoundError(f"No .safetensors files found in {path}")- for file in safetensor_files:- with safe_open(file, "pt", "cpu", writable_tensors=False) as f:- for weight_name in f.keys():- tensor = f.get_tensor(weight_name)- _apply_weight(model, packed_modules_mapping, weight_name, tensor)- _finalize_quantized_modules(model)- return- for file in weight_store.files:- file_weights = weight_store._file_to_weight_names.get(file, [])- if not file_weights:- continue- with safe_open(file, "pt", "cuda", writable_tensors = False, streaming = True) as f:- file_weights = f.reorder_for_streaming(file_weights)- for weight_name in file_weights:- tensor = f.get_tensor(weight_name)- if clone_loaded_tensors:- tensor = tensor.clone()- try:- _apply_weight(model, packed_modules_mapping, weight_name, tensor)- finally:- del tensor- f = None- if torch.cuda.is_available():- torch.cuda.synchronize()- gc.collect()- _finalize_quantized_modules(model)---def _apply_weight(model: nn.Module, packed_modules_mapping, weight_name: str, tensor: torch.Tensor):- quant_suffix = None- if weight_name.endswith(".weight._data"):- quant_suffix = "qdata"- elif weight_name.endswith(".weight._scale"):- quant_suffix = "qscale"- elif weight_name.endswith(".input_scale"):- quant_suffix = "input_scale"- elif weight_name.endswith(".output_scale"):- quant_suffix = "output_scale"-- mapped_name = weight_name- shard_id = None- for src_name, mapped in packed_modules_mapping.items():- if src_name in weight_name:- mapped_target = mapped[0] if isinstance(mapped, tuple) else mapped- mapped_name = weight_name.replace(src_name, mapped_target)- shard_id = mapped[1] if isinstance(mapped, tuple) and len(mapped) > 1 else None- break-- if quant_suffix is not None:- if quant_suffix == "qdata":- module_name = mapped_name[:-len(".weight._data")]- loader_name = "quant_weight_data_loader"- elif quant_suffix == "qscale":- module_name = mapped_name[:-len(".weight._scale")]- loader_name = "quant_weight_scale_loader"- elif quant_suffix == "input_scale":- module_name = mapped_name[:-len(".input_scale")]- loader_name = "quant_input_scale_loader"- else:- module_name = mapped_name[:-len(".output_scale")]- loader_name = "quant_output_scale_loader"- module = _get_submodule_safe(model, module_name)- if module is None:- print(f"[loader] Warning: Module not found: {module_name}")- return- loader_fn = getattr(module, loader_name, None)- if loader_fn is None:- print(f"[loader] Warning: Quant loader not found on module: {module_name}")- return- if shard_id is not None and quant_suffix in ("qdata", "qscale"):- loader_fn(tensor, shard_id)- else:- loader_fn(tensor)- return-- target = _get_parameter_target(model, mapped_name)- if target is None:- print(f"[loader] Warning: Parameter not found: {mapped_name}")- return- parent, param_name, param = target- weight_loader = getattr(param, "weight_loader", None)-- if shard_id is not None and weight_loader is not None:- if param.is_meta:- replacement_device = (- param.device if param.device.type != "meta" else torch.device("cuda" if torch.cuda.is_available() else "cpu")- )- replacement = torch.empty(param.shape, device=replacement_device, dtype=param.dtype)- if _should_replace_param_globally(model, param):- param = _replace_tied_meta_parameter(model, param, replacement)- else:- param = _replace_meta_parameter(parent, param_name, replacement, param)- try:- weight_loader(param, tensor, shard_id)- except TypeError:- weight_loader(param, tensor)- return-- if param.is_meta:- loaded_weight = _coerce_loaded_weight_to_param(param, tensor)- if loaded_weight.shape != param.shape:- raise RuntimeError(- f"Weight shape mismatch for '{mapped_name}': expected {tuple(param.shape)}, got {tuple(loaded_weight.shape)}"- )- if _should_replace_param_globally(model, param):- _replace_tied_meta_parameter(model, param, loaded_weight)- else:- _replace_meta_parameter(parent, param_name, loaded_weight, param)- return- if weight_loader is None:- weight_loader = default_weight_loader- weight_loader(param, tensor)---def _finalize_quantized_modules(model: nn.Module):- for module in model.modules():- finalize = getattr(module, "finalize_quantized", None)- if callable(finalize):- finalize()diff --git a/shared/llm_engines/nanovllm/vllm_support.py b/shared/llm_engines/nanovllm/vllm_support.pyindex f8c195e..7434544 100644--- a/shared/llm_engines/nanovllm/vllm_support.py+++ b/shared/llm_engines/nanovllm/vllm_support.py@@ -1,3 +1,5 @@+from typing import Any, Callable, Optional+_PROBE_CACHE = None_WARNED_REQUESTED_VLLM_UNAVAILABLE = False@@ -31,61 +33,6 @@ def _check_flash_attention_2():return True, "ok"-def _load_linear_module():- import importlib.util- import os-- base_dir = os.path.dirname(os.path.abspath(__file__))- linear_path = os.path.join(base_dir, "layers", "linear.py")- if not os.path.isfile(linear_path):- raise RuntimeError(f"Missing nanovllm linear kernel file: {linear_path}")-- spec = importlib.util.spec_from_file_location("nanovllm_linear_probe", linear_path)- if spec is None or spec.loader is None:- raise RuntimeError("Unable to build import spec for nanovllm linear kernel probe")- module = importlib.util.module_from_spec(spec)- spec.loader.exec_module(module)- return module---def _check_triton_int8_kernel():- import inspect- import torch-- if not torch.cuda.is_available():- return False, "CUDA is not available"-- try:- linear_module = _load_linear_module()- if not getattr(linear_module, "_TRITON_AVAILABLE", False):- return False, "nanovllm Triton path is disabled"-- run_kernel = getattr(linear_module, "_run_triton_fused_int8_mm", None)- if run_kernel is None:- return False, "nanovllm Triton int8 kernel entrypoint is missing"-- device = torch.device("cuda")- x = torch.randn((2, 64), device=device, dtype=torch.bfloat16)- qweight_t = torch.randint(-127, 128, (64, 32), device=device, dtype=torch.int8)- qweight_scale = torch.ones((32,), device=device, dtype=torch.float32)- # Support both legacy (4-arg) and current (3-arg) probe signatures.- param_count = len(inspect.signature(run_kernel).parameters)- if param_count >= 4:- out = run_kernel(x, qweight_t, qweight_scale, 0.01)- else:- out = run_kernel(x, qweight_t, qweight_scale)- torch.cuda.synchronize()-- if tuple(out.shape) != (2, 32):- return False, f"Unexpected kernel output shape: {tuple(out.shape)}"- if not torch.isfinite(out).all().item():- return False, "Kernel output contains non-finite values"- except Exception as exc:- return False, f"Triton int8 kernel smoke test failed: {exc}"-- return True, "ok"--def probe_vllm_runtime(force=False):global _PROBE_CACHEif _PROBE_CACHE is not None and not force:@@ -99,10 +46,7 @@ def probe_vllm_runtime(force=False):flash_ok, flash_msg = _check_flash_attention_2()checks["flash_attention_2"] = {"ok": flash_ok, "message": flash_msg}- kernel_ok, kernel_msg = _check_triton_int8_kernel()- checks["triton_int8_kernel"] = {"ok": kernel_ok, "message": kernel_msg}-- supported = triton_ok and flash_ok and kernel_ok+ supported = triton_ok and flash_okresult = {"supported": supported,"preferred_engine": "vllm" if supported else "legacy",@@ -114,6 +58,7 @@ def probe_vllm_runtime(force=False):def resolve_lm_decoder_engine(requested_engine):+ requested_engine = str(requested_engine or "").strip().lower()probe_result = probe_vllm_runtime()supported = bool(probe_result.get("supported", False))if requested_engine == "vllm":@@ -136,4 +81,264 @@ def resolve_lm_decoder_engine(requested_engine):return "legacy"if requested_engine == "":return "vllm" if supported else "legacy"- return requested_engine+ if requested_engine == "legacy":+ return "legacy"+ print(f"[LM] Unknown decoder engine '{requested_engine}', falling back to 'legacy'.")+ return "legacy"+++class NanoVllmTextEngine:+ keep_loaded_for_phase2 = True++ def __init__(self, model, model_path: str, tokenizer):+ self.model = model+ self.model_path = model_path+ self.tokenizer = tokenizer+ self._llm = None+ self._sampling_params_cls = None+ self._max_model_len_hint = None+ self._max_num_seqs_hint = None+ self._max_num_batched_tokens_hint = None+ self._last_failure_reason = ""++ @staticmethod+ def _compute_runtime_hints(prompt_len: int, max_tokens: int, cfg_scale: float):+ max_model_len = max(8, int(prompt_len) + int(max_tokens))+ max_num_seqs = 2 if cfg_scale and cfg_scale > 1.0 else 1+ max_num_batched_tokens = max_model_len * max_num_seqs+ return max_model_len, max_num_seqs, max_num_batched_tokens++ def _ensure_runtime_capacity(self, max_model_len: int, max_num_seqs: int, max_num_batched_tokens: int):+ if self._max_model_len_hint is None:+ self._max_model_len_hint = max_model_len+ self._max_num_seqs_hint = max_num_seqs+ self._max_num_batched_tokens_hint = max_num_batched_tokens+ return++ need_grow = (+ max_model_len > int(self._max_model_len_hint)+ or max_num_seqs > int(self._max_num_seqs_hint)+ or max_num_batched_tokens > int(self._max_num_batched_tokens_hint)+ )+ if not need_grow:+ return++ self._max_model_len_hint = max_model_len+ self._max_num_seqs_hint = max_num_seqs+ self._max_num_batched_tokens_hint = max_num_batched_tokens+ self.close()++ def reserve_runtime(self, prompt_len: int, max_tokens: int, cfg_scale: float):+ req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(+ prompt_len=prompt_len,+ max_tokens=max_tokens,+ cfg_scale=cfg_scale,+ )+ self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)++ def _ensure_llm(self):+ if self._llm is not None:+ return+ try:+ import torch._inductor.config as inductor_config++ if bool(getattr(inductor_config, "split_reductions", False)):+ inductor_config.split_reductions = False+ except Exception:+ pass+ try:+ from shared.llm_engines.nanovllm import LLM, SamplingParams+ except Exception as exc: # pragma: no cover+ raise RuntimeError(f"nano-vllm is not available for vllm engine: {exc}") from exc+ if not self.model_path:+ raise RuntimeError("vllm engine requires a model_path")+ max_model_len = self._max_model_len_hint or 4096+ max_num_seqs = self._max_num_seqs_hint or 1+ max_num_batched_tokens = self._max_num_batched_tokens_hint or (max_model_len * max_num_seqs)+ self._llm = LLM(+ model=self.model_path,+ enforce_eager=False,+ tensor_parallel_size=1,+ max_model_len=max_model_len,+ max_num_seqs=max_num_seqs,+ max_num_batched_tokens=max_num_batched_tokens,+ tokenizer=self.tokenizer,+ model_object=self.model,+ )+ self._sampling_params_cls = SamplingParams++ def release_runtime_allocations(self):+ if self._llm is None:+ return+ try:+ self._llm.reset_runtime_state()+ except Exception:+ pass++ def close(self):+ llm = getattr(self, "_llm", None)+ self._llm = None+ if llm is not None:+ try:+ close_fn = getattr(llm, "close", None)+ if callable(close_fn):+ close_fn()+ else:+ try:+ llm.reset_runtime_state()+ except Exception:+ pass+ try:+ llm.clear_graph_cache()+ except Exception:+ pass+ exit_fn = getattr(llm, "exit", None)+ if callable(exit_fn):+ exit_fn()+ except Exception:+ pass+ try:+ del llm+ except Exception:+ pass+ self._sampling_params_cls = None++ def __del__(self):+ self.close()++ @staticmethod+ def _extract_text_and_tokens(output_obj) -> tuple[str, list[int]]:+ if output_obj is None:+ return "", []+ if isinstance(output_obj, dict):+ text = str(output_obj.get("text", "") or "")+ token_ids = output_obj.get("token_ids", []) or []+ return text, [int(x) for x in token_ids]+ if hasattr(output_obj, "outputs"):+ outputs = getattr(output_obj, "outputs", None)+ if outputs and len(outputs) > 0:+ text = str(getattr(outputs[0], "text", "") or "")+ token_ids = getattr(outputs[0], "token_ids", None)+ if token_ids is None:+ token_ids = getattr(outputs[0], "token_ids_list", []) or []+ return text, [int(x) for x in token_ids] if token_ids else []+ text = str(getattr(output_obj, "text", "") or "")+ token_ids = getattr(output_obj, "token_ids", None)+ if token_ids is None:+ token_ids = []+ return text, [int(x) for x in token_ids]++ def get_last_failure_reason(self) -> str:+ return self._last_failure_reason++ def generate_text(+ self,+ prompt: str,+ prompt_negative: str,+ max_tokens: int,+ temperature: Optional[float],+ top_p: Optional[float],+ top_k: Optional[int],+ cfg_scale: float,+ seed: Optional[int],+ callback=None,+ abort_fn: Optional[Callable[[], bool]] = None,+ logits_processor: Optional[Any] = None,+ logits_processor_update_state: Optional[Callable[[int], None]] = None,+ stop_checker: Optional[Callable[[list[int], int], bool]] = None,+ progress_label: str = "LM text",+ release_vram_after: bool = True,+ ignore_eos: bool = False,+ ):+ del stop_checker+ if abort_fn is not None and abort_fn():+ if release_vram_after:+ self.release_runtime_allocations()+ return None+ try:+ prompt_len = len(self.tokenizer.encode(prompt))+ except Exception:+ prompt_len = 0+ if cfg_scale > 1.0 and prompt_negative:+ try:+ prompt_len = max(prompt_len, len(self.tokenizer.encode(prompt_negative)))+ except Exception:+ pass++ req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(+ prompt_len=prompt_len,+ max_tokens=max_tokens,+ cfg_scale=cfg_scale,+ )+ self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)+ self._ensure_llm()+ if self._llm is None:+ return None+ try:+ self._llm.reset()+ except Exception:+ pass++ if callback is not None:+ callback(+ step_idx=-1,+ override_num_inference_steps=max_tokens,+ denoising_extra=f"{progress_label} 0/{max_tokens}",+ progress_unit="tokens",+ )++ seed_value = None+ if seed is not None:+ try:+ seed_value = int(seed)+ except Exception:+ seed_value = None+ if seed_value is not None and seed_value < 0:+ seed_value = None++ temp = temperature if temperature is not None and temperature > 0 else 1e-5+ sampling_params = self._sampling_params_cls(+ temperature=temp,+ max_tokens=max_tokens,+ cfg_scale=max(cfg_scale, 1.0),+ top_k=top_k if top_k is not None and top_k > 0 else None,+ top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,+ ignore_eos=bool(ignore_eos),+ logits_processor=logits_processor,+ logits_processor_update_state=logits_processor_update_state,+ seed=seed_value,+ )++ text = ""+ token_ids: list[int] = []+ try:+ outputs = self._llm.generate(+ prompts=[prompt],+ sampling_params=sampling_params,+ use_tqdm=True,+ unconditional_prompts=[prompt_negative] if cfg_scale > 1.0 else None,+ )+ if outputs:+ text, token_ids = self._extract_text_and_tokens(outputs[0])+ if (not text) and token_ids:+ try:+ text = self.tokenizer.decode(token_ids, skip_special_tokens=False)+ except Exception:+ text = ""+ self._last_failure_reason = ""+ except Exception as exc:+ self._last_failure_reason = str(exc)+ raise+ finally:+ if release_vram_after:+ self.release_runtime_allocations()++ if callback is not None:+ callback(+ step_idx=max(0, max_tokens - 1),+ override_num_inference_steps=max_tokens,+ denoising_extra=f"{progress_label} {max_tokens}/{max_tokens}",+ progress_unit="tokens",+ )++ return {"token_ids": token_ids, "text": text}