Python
added Qwen3.5VL Abliterated Prompt Enhancer + GGUF Cuda Kernels support
DeepBeepMeepc65a4496df3f
diff --git a/defaults/ltx2_22B_distilled_gguf_q4_k_m.json b/defaults/ltx2_22B_distilled_gguf_q4_k_m.jsonnew file mode 100644index 0000000..142ba11--- /dev/null+++ b/defaults/ltx2_22B_distilled_gguf_q4_k_m.json@@ -0,0 +1,16 @@+{+ "model": {+ "name": "LTX-2 2.3 Distilled GGUF Q4_K_M Light 22B",+ "architecture": "ltx2_22B",+ "description": "LTX-2.3 distilled GGUF Q4_K_M light checkpoint (22B) without bundled AV connector weights.",+ "URLs": [+ "https://huggingface.co/DeepBeepMeep/LTX-2/resolve/main/ltx-2.3-22b-distilled-Q4_K_M_light.gguf"+ ],+ "preload_URLs": "ltx2_22B",+ "ltx2_pipeline": "distilled"+ },+ "prompt": "A warm sunny backyard. The camera starts in a tight cinematic close-up of a woman and a man in their 30s, facing each other with serious expressions. The woman, emotional and dramatic, says softly, \"That's it... Dad's lost it. And we've lost Dad.\"The man exhales, slightly annoyed: \"Stop being so dramatic, Jess.\"A beat. He glances aside, then mutters defensively, \"He's just having fun.\"The camera slowly pans right, revealing the grandfather in the garden wearing enormous butterfly wings, waving his arms in the air like he's trying to take off.He shouts, \"Wheeeew!\" as he flaps his wings with full commitment.The woman covers her face, on the verge of tears. The tone is deadpan, absurd, and quietly tragic.",+ "num_inference_steps": 8,+ "video_length": 241,+ "resolution": "1280x720"+}diff --git a/defaults/ltx2_22B_distilled_gguf_q6_k.json b/defaults/ltx2_22B_distilled_gguf_q6_k.jsonnew file mode 100644index 0000000..d88567a--- /dev/null+++ b/defaults/ltx2_22B_distilled_gguf_q6_k.json@@ -0,0 +1,16 @@+{+ "model": {+ "name": "LTX-2 2.3 Distilled GGUF Q6_K Light 22B",+ "architecture": "ltx2_22B",+ "description": "LTX-2.3 distilled GGUF Q6_K light checkpoint (22B) without bundled AV connector weights.",+ "URLs": [+ "https://huggingface.co/DeepBeepMeep/LTX-2/resolve/main/ltx-2.3-22b-distilled-Q6_K_light.gguf"+ ],+ "preload_URLs": "ltx2_22B",+ "ltx2_pipeline": "distilled"+ },+ "prompt": "A warm sunny backyard. The camera starts in a tight cinematic close-up of a woman and a man in their 30s, facing each other with serious expressions. The woman, emotional and dramatic, says softly, \"That's it... Dad's lost it. And we've lost Dad.\"The man exhales, slightly annoyed: \"Stop being so dramatic, Jess.\"A beat. He glances aside, then mutters defensively, \"He's just having fun.\"The camera slowly pans right, revealing the grandfather in the garden wearing enormous butterfly wings, waving his arms in the air like he's trying to take off.He shouts, \"Wheeeew!\" as he flaps his wings with full commitment.The woman covers her face, on the verge of tears. The tone is deadpan, absurd, and quietly tragic.",+ "num_inference_steps": 8,+ "video_length": 241,+ "resolution": "1280x720"+}diff --git a/defaults/ltx2_22B_distilled_gguf_q8_0.json b/defaults/ltx2_22B_distilled_gguf_q8_0.jsonnew file mode 100644index 0000000..848a896--- /dev/null+++ b/defaults/ltx2_22B_distilled_gguf_q8_0.json@@ -0,0 +1,16 @@+{+ "model": {+ "name": "LTX-2 2.3 Distilled GGUF Q8_0 Light 22B",+ "architecture": "ltx2_22B",+ "description": "LTX-2.3 distilled GGUF Q8_0 light checkpoint (22B) without bundled AV connector weights.",+ "URLs": [+ "https://huggingface.co/DeepBeepMeep/LTX-2/resolve/main/ltx-2.3-22b-distilled-Q8_0_light.gguf"+ ],+ "preload_URLs": "ltx2_22B",+ "ltx2_pipeline": "distilled"+ },+ "prompt": "A warm sunny backyard. The camera starts in a tight cinematic close-up of a woman and a man in their 30s, facing each other with serious expressions. The woman, emotional and dramatic, says softly, \"That's it... Dad's lost it. And we've lost Dad.\"The man exhales, slightly annoyed: \"Stop being so dramatic, Jess.\"A beat. He glances aside, then mutters defensively, \"He's just having fun.\"The camera slowly pans right, revealing the grandfather in the garden wearing enormous butterfly wings, waving his arms in the air like he's trying to take off.He shouts, \"Wheeeew!\" as he flaps his wings with full commitment.The woman covers her face, on the verge of tears. The tone is deadpan, absurd, and quietly tragic.",+ "num_inference_steps": 8,+ "video_length": 241,+ "resolution": "1280x720"+}diff --git a/defaults/ltx2_distilled_gguf_q4_k_m.json b/defaults/ltx2_distilled_gguf_q4_k_m.jsonindex 60863ab..a007fae 100644--- a/defaults/ltx2_distilled_gguf_q4_k_m.json+++ b/defaults/ltx2_distilled_gguf_q4_k_m.json@@ -2,7 +2,7 @@"model": {"name": "LTX-2 2.0 Distilled GGUF Q4_K_M 19B ","architecture": "ltx2_19B",- "description": "LTX-2 distilled GGUF Q4_K_M checkpoint for llama.cpp-backed quantization.",+ "description": "LTX-2 distilled GGUF Q4_K_M checkpoint made by Kijai.","URLs": ["https://huggingface.co/Kijai/LTXV2_comfy/resolve/main/diffusion_models/ltx-2-19b-distilled_Q4_K_M.gguf"],diff --git a/defaults/ltx2_distilled_gguf_q6_k.json b/defaults/ltx2_distilled_gguf_q6_k.jsonindex cf5ac75..42d7ad6 100644--- a/defaults/ltx2_distilled_gguf_q6_k.json+++ b/defaults/ltx2_distilled_gguf_q6_k.json@@ -2,7 +2,7 @@"model": {"name": "LTX-2 2.0 Distilled GGUF Q6_K 19B ","architecture": "ltx2_19B",- "description": "LTX-2 distilled GGUF Q6_K checkpoint for llama.cpp-backed quantization.",+ "description": "LTX-2 distilled GGUF Q6_K checkpoint made by Kijai.","URLs": ["https://huggingface.co/Kijai/LTXV2_comfy/resolve/main/diffusion_models/ltx-2-19b-distilled_Q6_K.gguf"],diff --git a/defaults/ltx2_distilled_gguf_q8_0.json b/defaults/ltx2_distilled_gguf_q8_0.jsonindex b253429..d875aa0 100644--- a/defaults/ltx2_distilled_gguf_q8_0.json+++ b/defaults/ltx2_distilled_gguf_q8_0.json@@ -2,7 +2,7 @@"model": {"name": "LTX-2 2.0 Distilled GGUF Q8_0 19B ","architecture": "ltx2_19B",- "description": "LTX-2 distilled GGUF Q8_0 checkpoint for llama.cpp-backed quantization.",+ "description": "LTX-2 distilled GGUF Q8_0 checkpoint made by Kijai.","URLs": ["https://huggingface.co/Kijai/LTXV2_comfy/resolve/main/diffusion_models/ltx-2-19b-distilled_Q8_0.gguf"],diff --git a/docs/API.md b/docs/API.mdnew file mode 100644index 0000000..8b97c00--- /dev/null+++ b/docs/API.md@@ -0,0 +1,476 @@+# WanGP Python API++`shared/api.py` provides a lightweight in-process wrapper over WanGP's existing generation path.++The main goal is to let third-party code call WanGP directly, keep the last loaded model alive across requests, receive structured progress updates, and still capture the same stdout/stderr output that would normally go to the console.++## Quick Start++```python+from pathlib import Path++from shared.api import init++session = init(+ root=Path(r"C:\WanGP"),+ cli_args=["--attention", "sdpa", "--profile", "4"],+)++settings = {+ "model_type": "ltx2_22B_distilled",+ "prompt": "Cinematic shot of a neon train entering a rainy station",+ "resolution": "1280x704",+ "num_inference_steps": 8,+ "video_length": 97,+ "duration_seconds": 4,+ "force_fps": 24,+}++job = session.submit_task(settings)++for event in job.events.iter(timeout=0.2):+ if event.kind == "progress":+ progress = event.data+ print(progress.phase, progress.progress, progress.current_step, progress.total_steps)+ elif event.kind == "preview":+ preview = event.data+ if preview.image is not None:+ preview.image.save("preview.png")+ elif event.kind == "stream":+ line = event.data+ print(f"[{line.stream}] {line.text}")++result = job.result()+if result.success:+ print(result.generated_files)+else:+ for error in result.errors:+ print(error.message)+```++## Main Entry Points++- `init(...) -> WanGPSession`+ - Creates a reusable session and eagerly loads the runtime.+- `WanGPSession.submit(source) -> SessionJob`+ - Starts a job from a settings dict, a manifest list, or a saved `.json` / `.zip` file.+- `WanGPSession.submit_task(settings) -> SessionJob`+ - Preferred single-task entrypoint.+- `WanGPSession.submit_manifest(settings_list) -> SessionJob`+ - Batch entrypoint for multiple tasks.+- `SessionJob.result() -> GenerationResult`+ - Waits for completion and returns a structured result object.+- `SessionJob.cancel()`+ - Requests cancellation of the active generation.++## `init(...)` Parameters++```python+session = init(+ root=Path(r"C:\WanGP"),+ config_path=Path(r"C:\WanGP\wgp_config.json"), # optional+ output_dir=Path(r"C:\WanGP\outputs_override"), # optional+ callbacks=MyCallbacks(), # optional+ cli_args=["--attention", "sdpa"], # optional+ console_output=True, # optional, default=True+)+```++- `root`+ - Path to the WanGP installation folder.+ - Example: `C:\WanGP`++- `config_path`+ - Optional path to `wgp_config.json`.+ - If omitted, WanGP uses `C:\WanGP\wgp_config.json`.+ - This must point to a file named `wgp_config.json`.++- `output_dir`+ - Optional override for generated outputs.+ - If omitted, WanGP uses the output paths defined in the config file.++- `callbacks`+ - Optional callback object. See the callback section below.++- `cli_args`+ - Optional WanGP startup flags.+ - Example: `["--attention", "sdpa", "--profile", "4"]`++- `console_output`+ - Enables or disables writing WanGP stdout/stderr to the real console.+ - Default: `True`+ - The stream object always receives a copy of stdout/stderr, regardless of this setting.++## Accepted Input Shapes++Relative attachment paths are normalized to absolute paths when the job is submitted.++- For direct settings dictionaries and `.json` settings files, the base is the API caller's current working directory at submit time.+- For `.zip` queue files, WanGP keeps the queue bundle behavior and resolves bundled media from the extracted queue contents.+- A few WanGP string-like fields are normalized for convenience. For example, `force_fps` may be passed as `24` or `"24"`.++### Single Task++For single-task use, the intended input is the task settings dictionary itself:++```python+settings = {+ "model_type": "qwen_image_20B",+ "prompt": "A red bicycle parked in front of a bakery",+ "resolution": "1024x1024",+ "num_inference_steps": 4,+ "image_mode": 1,+}++job = session.submit_task(settings)+```++### Manifest++`submit_manifest(...)` accepts a list of settings dictionaries:++```python+settings_list = [+ {+ "model_type": "qwen_image_20B",+ "prompt": "A quiet library at sunrise",+ "resolution": "1024x1024",+ "num_inference_steps": 4,+ "image_mode": 1,+ },+ {+ "model_type": "qwen_image_20B",+ "prompt": "A rainy alley with neon signs",+ "resolution": "1024x1024",+ "num_inference_steps": 4,+ "image_mode": 1,+ },+]++job = session.submit_manifest(settings_list)+```++### Saved Queue / Settings File++`submit(...)` also accepts:++- a `.json` settings file path+- a `.zip` saved queue path++Example:++```python+job = session.submit(Path(r"C:\WanGP\my_queue.zip"))+```++## Streaming Events++Each job exposes `job.events`, a `SessionStream`.++The stream yields `SessionEvent` objects:++```python+SessionEvent(+ kind="progress",+ data=ProgressUpdate(...),+ timestamp=1710000000.0,+)+```++Known `kind` values:++- `started`+ - Job accepted and session processing started.+- `progress`+ - Structured progress update.+- `preview`+ - RGB preview update.+- `stream`+ - One stdout/stderr line.+- `status`+ - WanGP status message.+- `info`+ - WanGP informational message.+- `output`+ - Raw output refresh event from WanGP.+- `refresh_models`+ - Raw model-refresh event from WanGP.+- `completed`+ - Final `GenerationResult`.+- `error`+ - One `GenerationError` record.++## Returned Objects++### `GenerationResult`++Returned by `job.result()`:++```python+GenerationResult(+ success=False,+ generated_files=[+ r"C:\WanGP\outputs\clip_001.mp4",+ ],+ errors=[+ GenerationError(+ message="Task 2 failed validation",+ task_index=2,+ task_id=2,+ stage="validation",+ ),+ ],+ total_tasks=3,+ successful_tasks=2,+ failed_tasks=1,+)+```++Fields:++- `success: bool`+ - `True` only when every submitted task completed without error.+- `generated_files: list[str]`+ - Absolute paths to every file generated by the job, including partial-success runs.+- `errors: list[GenerationError]`+ - Structured error records collected during the run.+- `total_tasks: int`+ - Number of tasks submitted in the job.+- `successful_tasks: int`+ - Number of tasks that completed successfully.+- `failed_tasks: int`+ - Number of tasks that failed or were cancelled.++`job.result()` does not raise generation-task failures. Instead, inspect `result.success` and `result.errors`.++### `GenerationError`++Delivered through `error` events, `on_error(...)`, and `GenerationResult.errors`:++```python+GenerationError(+ message="Task 2 did not complete successfully",+ task_index=2,+ task_id=2,+ stage="generation",+)+```++Fields:++- `message: str`+ - Human-readable error message.+- `task_index: int | None`+ - One-based task index when the error is associated with a specific task.+- `task_id: Any`+ - Task identifier from the manifest when available.+- `stage: str | None`+ - Error stage such as `validation`, `generation`, `cancelled`, or `runtime`.++### `ProgressUpdate`++Delivered through `progress` events and `on_progress(...)`:++```python+ProgressUpdate(+ phase="inference",+ status="Prompt 1/1 | Denoising | 7.2s",+ progress=54,+ current_step=4,+ total_steps=8,+ raw_phase="Denoising",+ unit=None,+)+```++Fields:++- `phase: str`+ - Normalized phase. Typical values:+ - `loading_model`+ - `encoding_text`+ - `inference`+ - `decoding`+ - `downloading_output`+ - `cancelled`+- `status: str`+ - Human-readable status string produced by WanGP.+- `progress: int`+ - Estimated percentage from `0` to `100`.+- `current_step: int | None`+ - Current inference step when available.+- `total_steps: int | None`+ - Total inference steps when available.+- `raw_phase: str | None`+ - Original WanGP phase label before normalization.+- `unit: str | None`+ - Optional progress unit if WanGP provides one.++### `PreviewUpdate`++Delivered through `preview` events and `on_preview(...)`:++```python+PreviewUpdate(+ image=<PIL.Image.Image image mode=RGB size=800x200>,+ phase="inference",+ status="Prompt 1/1 | Denoising",+ progress=54,+ current_step=4,+ total_steps=8,+)+```++Fields:++- `image: PIL.Image.Image | None`+ - RGB preview image generated from WanGP's latent preview payload.+- `phase`, `status`, `progress`, `current_step`, `total_steps`+ - Same interpretation as `ProgressUpdate`.++### `StreamMessage`++Delivered through `stream` events and `on_stream(...)`:++```python+StreamMessage(+ stream="stdout",+ text="New video saved to Path: C:\\WanGP\\outputs\\clip_001.mp4",+)+```++Fields:++- `stream: str`+ - Usually `stdout` or `stderr`.+- `text: str`+ - One redirected line of console output.++### `SessionEvent`++Generic event wrapper:++```python+SessionEvent(+ kind="stream",+ data=StreamMessage(stream="stdout", text="Model loaded"),+ timestamp=1710000000.0,+)+```++Fields:++- `kind: str`+ - Event type.+- `data: Any`+ - Payload object for that event.+- `timestamp: float`+ - Event creation time.++## Callback Object++You can pass a callback object to `init(...)` or `WanGPSession(...)`.++Supported callback methods:++- `on_progress(progress_update)`+ - Called when WanGP emits a structured progress update.+ - Use this for progress bars, step counters, and status text.++- `on_preview(preview_update)`+ - Called when a preview image is available.+ - Use this when you want live RGB preview frames during inference.++- `on_stream(stream_message)`+ - Called for every redirected stdout/stderr line.+ - This is the programmatic equivalent of watching the terminal output.++- `on_status(text)`+ - Called for WanGP status messages.+ - Use this if you want coarse status without parsing full progress objects.++- `on_info(text)`+ - Called for informational messages.++- `on_output(data)`+ - Called for raw WanGP output refresh events.+ - This is a low-level hook and is usually not needed by third-party integrations.++- `on_complete(result)`+ - Called when the job finishes.+ - Receives a `GenerationResult`.++- `on_error(error)`+ - Called each time WanGP reports a task or runtime error.+ - Receives a `GenerationError`.++- `on_event(session_event)`+ - Generic catch-all event hook.+ - Called alongside the specific callback above, not instead of it.++Example:++```python+class Callbacks:+ def on_progress(self, progress):+ print("progress:", progress.progress, progress.phase)++ def on_preview(self, preview):+ if preview.image is not None:+ preview.image.save("latest_preview.png")++ def on_stream(self, line):+ print(f"[{line.stream}] {line.text}")++ def on_complete(self, result):+ print("success:", result.success)+ print("generated:", result.generated_files)++ def on_error(self, error):+ print("error:", error.message)+```++Full signature example:++```python+from shared.api import GenerationError, GenerationResult, PreviewUpdate, ProgressUpdate, SessionEvent, StreamMessage+++class VerboseCallbacks:+ def on_progress(self, progress: ProgressUpdate) -> None:+ print("progress", progress.progress, progress.current_step, progress.total_steps)++ def on_preview(self, preview: PreviewUpdate) -> None:+ print("preview", preview.phase, preview.image.size if preview.image is not None else None)++ def on_stream(self, line: StreamMessage) -> None:+ print(line.stream, line.text)++ def on_status(self, text: str) -> None:+ print("status", text)++ def on_info(self, text: str) -> None:+ print("info", text)++ def on_output(self, data: object) -> None:+ print("output", data)++ def on_complete(self, result: GenerationResult) -> None:+ print("success", result.success)+ print("files", result.generated_files)++ def on_error(self, error: GenerationError) -> None:+ print("error", error.stage, error.task_index, error.message)++ def on_event(self, event: SessionEvent) -> None:+ print("event", event.kind)+```++## Cancellation++```python+job = session.submit_task(settings)+job.cancel()+```++Cancellation is cooperative and forwards WanGP's normal abort signal to the active model. A cancelled run completes with `result.success == False` and a cancellation entry in `result.errors`.diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.mdindex bb6af05..22457d1 100644--- a/docs/INSTALLATION.md+++ b/docs/INSTALLATION.md@@ -152,12 +152,12 @@ pip install -r requirements.txt### Flash Attention Windows-#### Pytorch 2.7.1+#### Pytorch 2.7.1 / Python 3.10```pip install https://github.com/Redtash1/Flash_Attention_2_Windows/releases/download/v2.7.0-v2.7.4/flash_attn-2.7.4.post1+cu128torch2.7.0cxx11abiFALSE-cp310-cp310-win_amd64.whl```-#### Pytorch 2.10-https://github.com/deepbeepmeep/kernels/releases/download/Flash2/flash_attn-2.8.3-cp311-cp311-win_amd64.whl+#### Pytorch 2.10 / Python 3.11+pip install https://github.com/deepbeepmeep/kernels/releases/download/Flash2/flash_attn-2.8.3-cp311-cp311-win_amd64.whl@@ -284,7 +284,7 @@ pip install flash-attn==2.7.2.post1### Attention GPU Compatibility- RTX 10XX: SDPA-- RTX 20XX: SPDA, Sage1+- RTX 20XX: SDPA, Sage1- RTX 30XX, 40XX: SDPA, Flash Attention, Xformers, Sage1, Sage2/Sage2++- RTX 50XX: SDPA, Flash Attention, Xformers, Sage2/Sage2++ / Sage3@@ -318,7 +318,38 @@ If Sage attention doesn't work:For more troubleshooting, see [TROUBLESHOOTING.md](TROUBLESHOOTING.md)-## Optional Kernels for INT4 / FP4 quantized support+## Optional Kernels++### GGUF llama.cpp CUDA Kernels++These kernels are only useful for GGUF models.++### GGUF Kernels Wheels for Python 3.11 / Pytorch 2.10 / Cuda 13++- Windows+ ```+ pip install https://github.com/deepbeepmeep/kernels/releases/download/GGUF_Kernels/llamacpp_gguf_cuda-1.0.2+torch210cu13py311-cp311-cp311-win_amd64.whl+ ```++- Linux+ ```+ # placeholder: Linux GGUF wheel for Python 3.11 / PyTorch 2.10 / CUDA 13 not published yet+ ```++### GGUF Kernels Wheels for Python 3.10 / Pytorch 2.7.1 / Cuda 12.8++- Windows+ ```+ # placeholder: Windows GGUF wheel for Python 3.10 / PyTorch 2.7.1 / CUDA 12.8 not published yet+ ```++- Linux+ ```+ # placeholder: Linux GGUF wheel for Python 3.10 / PyTorch 2.7.1 / CUDA 12.8 not published yet+ ```++### INT4 / FP4 quantized support+These kernels will offer optimized INT4 / FP4 dequantization.**Please Note FP4 support is hardware dependent and will work only with RTX 50xx / sm120+ GPUs**@@ -357,4 +388,4 @@ These kernels will offer optimized INT4 / FP4 dequantization.- Linux```pip install https://github.com/nunchaku-ai/nunchaku/releases/download/v1.2.1/nunchaku-1.2.1+cu13.0torch2.10-cp311-cp311-linux_x86_64.whl- ```\ No newline at end of file+ ```diff --git a/models/_settings.json b/models/_settings.jsonindex 7d0cd73..0d36e0f 100644--- a/models/_settings.json+++ b/models/_settings.json@@ -99,5 +99,6 @@"audio_guide": null,"audio_guide2": null,"custom_guide": null,- "audio_source": null+ "audio_source": null,+ "matanyone_version": "v1"}diff --git a/models/ltx2/configs/ltx2_22b_config.json b/models/ltx2/configs/ltx2_22b_config.jsonindex 1485786..7eec8c4 100644--- a/models/ltx2/configs/ltx2_22b_config.json+++ b/models/ltx2/configs/ltx2_22b_config.json@@ -250,7 +250,8 @@"out_channels": 3,"patch_size": 4,"scaling_factor": 1.0,- "spatial_padding_mode": "zeros",+ "encoder_spatial_padding_mode": "zeros",+ "decoder_spatial_padding_mode": "reflect","timestep_conditioning": false,"use_quant_conv": false},@@ -353,4 +354,4 @@"use_tanh_at_final": false}}-}\ No newline at end of file+}diff --git a/models/ltx2/ltx2.py b/models/ltx2/ltx2.pyindex 71553e0..4beec4f 100644--- a/models/ltx2/ltx2.py+++ b/models/ltx2/ltx2.py@@ -249,7 +249,10 @@ class _LTX2VAEHelper:mixed_precision: bool,output_height: int | None = None,output_width: int | None = None,- ) -> int:+ ) -> int | tuple[int, int]:+ if vae_config >= 4:+ vae_config = 0+if vae_config == 0:if mixed_precision:device_mem_capacity = device_mem_capacity / 1.5@@ -266,13 +269,15 @@ class _LTX2VAEHelper:if ref_size is not None and ref_size > 480:use_vae_config += 1+ spatial_tile_size = 128if use_vae_config <= 1:- return 0- if use_vae_config == 2:- return 512- if use_vae_config == 3:- return 256- return 128+ spatial_tile_size = 0+ elif use_vae_config == 2:+ spatial_tile_size = 512+ elif use_vae_config == 3:+ spatial_tile_size = 256++ return spatial_tile_sizedef _attach_lora_preprocessor(transformer: torch.nn.Module) -> None:@@ -376,11 +381,15 @@ def _normalize_temporal_overlap(overlap_frames: int, tile_frames: int) -> int:def _build_tiling_config(tile_size: int | tuple | list | None, fps: float | None) -> TilingConfig | None:+ temporal_tiling_divisor = 1spatial_config = Noneif isinstance(tile_size, (tuple, list)):if len(tile_size) == 0:tile_size = None- tile_size = tile_size[-1]+ else:+ if len(tile_size) > 1:+ temporal_tiling_divisor = max(1, int(tile_size[0] or 1))+ tile_size = tile_size[-1]if tile_size is not None:tile_size = _normalize_tiling_size(tile_size)if tile_size > 0:@@ -392,7 +401,8 @@ def _build_tiling_config(tile_size: int | tuple | list | None, fps: float | Nonetemporal_config = Noneif fps is not None and fps > 0:- tile_frames = _normalize_temporal_tiling_size(int(math.ceil(float(fps) * 5.0)))+ temporal_tiling_divisor = max(1, temporal_tiling_divisor)+ tile_frames = _normalize_temporal_tiling_size(int(math.ceil(float(fps) * 5.0 / temporal_tiling_divisor)))if tile_frames > 0:overlap_frames = int(round(tile_frames * 3 / 8))overlap_frames = _normalize_temporal_overlap(overlap_frames, tile_frames)@@ -568,7 +578,7 @@ class LTX2:transformer_sd_ops = LTXV_MODEL_COMFY_RENAMING_MAPwith init_empty_weights():velocity_model = LTXModelConfigurator.from_config(base_config)- velocity_model = _load_component(velocity_model, transformer_path, transformer_sd_ops)+ velocity_model = _load_component(velocity_model, transformer_path, transformer_sd_ops, ignore_unused_weights=True)transformer = X0Model(velocity_model)transformer.eval().requires_grad_(False)VAE_URLs = self.model_def.get("VAE_URLs", None)diff --git a/models/ltx2/ltx2_handler.py b/models/ltx2/ltx2_handler.pyindex 9aece38..302e41d 100644--- a/models/ltx2/ltx2_handler.py+++ b/models/ltx2/ltx2_handler.py@@ -145,6 +145,7 @@ class family_handler:"frames_steps": 8,"sliding_window": True,"image_prompt_types_allowed": "TSEV",+ "end_frames_always_enabled": True,"returns_audio": True,"any_audio_prompt": True,"audio_prompt_choices": True,diff --git a/models/wan/scail/__init__.py b/models/wan/scail/__init__.pyindex ac6b329..c75a8ba 100644--- a/models/wan/scail/__init__.py+++ b/models/wan/scail/__init__.py@@ -113,6 +113,7 @@ class ScailPoseProcessor:# Lazy-loaded for multi-person mode.self._sam_segmenter = Noneself._matanyone_model = None+ self._matanyone_version = Nonedef unload(self):"""Release NLF model from memory (RAM and VRAM)."""@@ -143,6 +144,7 @@ class ScailPoseProcessor:except Exception:passself._matanyone_model = None+ self._matanyone_version = None# Force garbage collection and empty CUDA cachegc.collect()@@ -157,11 +159,18 @@ class ScailPoseProcessor:self._sam_segmenter = BaseSegmenter(SAM_checkpoint=None, model_type="vit_h", device=f"cuda:{self.gpu_id}")def _ensure_matanyone_loaded(self) -> None:- if self._matanyone_model is not None:+ from preprocessing.matanyone.utils.model_assets import get_selected_matanyone_version, load_selected_matanyone_model++ selected_version = get_selected_matanyone_version()+ if self._matanyone_model is not None and self._matanyone_version == selected_version:return- from preprocessing.matanyone.matanyone.model.matanyone import MatAnyone+ if self._matanyone_model is not None:+ self._matanyone_model.to("cpu")+ self._matanyone_model = None+ torch.cuda.empty_cache()- self._matanyone_model = MatAnyone.from_pretrained(fl.locate_folder("mask")).eval()+ self._matanyone_model, self._matanyone_version, _ = load_selected_matanyone_model()+ self._matanyone_model = self._matanyone_model.eval()self._matanyone_model.to(f"cuda:{self.gpu_id}")def _extract_and_render_multi(@@ -257,6 +266,7 @@ class ScailPoseProcessor:self._matanyone_model.to("cpu")del self._matanyone_modelself._matanyone_model = None+ self._matanyone_version = Nonetorch.cuda.empty_cache()# 4) Run NLF per segmented person-video.diff --git a/plugins/wan2gp-configuration/plugin.py b/plugins/wan2gp-configuration/plugin.pyindex 45cbdad..1fc7fd4 100644--- a/plugins/wan2gp-configuration/plugin.py+++ b/plugins/wan2gp-configuration/plugin.py@@ -166,9 +166,19 @@ class ConfigTabPlugin(WAN2GPPlugin):self.VAE_precision_choice = gr.Dropdown(choices=[("16-bit (faster, less VRAM)", "16"), ("32-bit (slower, better for sliding window)", "32")], value=self.server_config.get("vae_precision", "16"), label="VAE Encoding/Decoding Precision")self.compile_choice = gr.Dropdown(choices=[("On (up to 20% faster, requires Triton)", "transformer"), ("Off", "")], value=self.compile, label="Compile Transformer Model (slight speed again, but first generation is slower and potential compatibility issues with some GPUs/Models)", interactive=not self.args.lock_config)self.depth_anything_v2_variant_choice = gr.Dropdown(choices=[("Large (more precise, slower)", "vitl"), ("Big (less precise, faster)", "vitb")], value=self.server_config.get("depth_anything_v2_variant", "vitl"), label="Depth Anything v2 VACE Preprocessor")- self.vae_config_choice = gr.Dropdown(choices=[("Auto", 0), ("Disabled (fastest, high VRAM)", 1), ("256x256 Tiles (for >=8GB VRAM)", 2), ("128x128 Tiles (for >=6GB VRAM)", 3)], value=self.vae_config, label="VAE Tiling (to reduce VRAM usage)")+ vae_config_value = self.vae_config if self.vae_config in [0, 1, 2, 3] else 0+ self.vae_config_choice = gr.Dropdown(+ choices=[+ ("Auto", 0),+ ("Spatial / Temporal Tiling Preset for 16GB+", 1),+ ("Spatial / Temporal Tiling Preset for 8GB+", 2),+ ("Spatial / Temporal Tiling Preset for 6GB+", 3),+ ],+ value=vae_config_value,+ label="VAE Tiling (higher presets use less VRAM and may increase artifacts like banding)",+ )self.boost_choice = gr.Dropdown(choices=[("ON", 1), ("OFF", 2)], value=self.boost, label="Boost (~10% speedup for ~1GB VRAM)")- self.enable_int8_kernels_choice = gr.Dropdown(choices=[("Disabled", 0), ("Enabled", 1)], value=self.server_config.get("enable_int8_kernels", 0), label="Int8 Kernels (Experimental, 10% faster with INT8 quantized checkpoints, requires Triton)")+ self.enable_int8_kernels_choice = gr.Dropdown(choices=[("Disabled", 0), ("Enabled if Triton availabe", 1)], value=self.server_config.get("enable_int8_kernels", 1), label="Int8 Kernels (Experimental, 10% faster with INT8 quantized checkpoints, requires Triton)")self.video_profile_choice = gr.Dropdown(choices=self.memory_profile_choices,value=self.default_profile_video,@@ -196,7 +206,12 @@ class ConfigTabPlugin(WAN2GPPlugin):with gr.Tab("Extensions"):with gr.Group():- self.enhancer_enabled_choice = gr.Dropdown(choices=[("Off", 0), ("Florence 2 (image captioning) + LLama 3.2 3B (text generation)", 1), ("Florence 2 (image captioning) + Llama Joy 8B (uncensored, richer)", 2)], value=self.server_config.get("enhancer_enabled", 0), label="Prompt Enhancer (requires 8-14GB extra download)")+ self.enhancer_enabled_choice = gr.Dropdown(choices=[("Off", 0), ("Florence 2 (image captioning) + LLama 3.2 3B (text generation)", 1), ("Florence 2 (image captioning) + Llama Joy 8B (uncensored, richer)", 2), ("Qwen3.5VL Abliterated 4B (captioning + uncensored text enhancement, vllm accelerated if available)", 3), ("Qwen3.5VL Abliterated 9B (captioning + uncensored high end text enhancement, vllm accelerated if available)", 4)], value=self.server_config.get("enhancer_enabled", 0), label="Prompt Enhancer (requires extra model files)")+ self.enhancer_quantization_choice = gr.Dropdown(+ choices=[("Quanto Int8 (better quality)", "quanto_int8"), ("GGUF Q4 (less VRAM/RAM & faster if kernels are installed, but worse quality)", "gguf")],+ value=self.server_config.get("prompt_enhancer_quantization", "quanto_int8"),+ label="Prompt Enhancer Qwen3.5 LLM Quantization",+ )self.enhancer_mode_choice = gr.Dropdown(choices=[("On-Demand Button Only", 1),("Automatic on Generation", 0)], value=self.server_config.get("enhancer_mode", 1), label="Prompt Enhancer Usage")with gr.Row():self.prompt_enhancer_temperature_choice = gr.Slider(@@ -245,11 +260,28 @@ class ConfigTabPlugin(WAN2GPPlugin):label="RIFE Temporal Upsampling Model",interactive=not self.args.lock_config)+ self.matanyone_version_choice = gr.Dropdown(+ choices=[("MatAnyone v1 (original, default)", "v1"), ("MatAnyone v2", "v2")],+ value=self.server_config.get("matanyone_version", "v1"),+ label="MatAnyone Video Mask Model",+ interactive=not self.args.lock_config+ )with gr.Tab("Outputs"):self.video_output_codec_choice = gr.Dropdown(choices=[("x265 CRF 28 (Balanced)", 'libx265_28'), ("x264 Level 8 (Balanced)", 'libx264_8'), ("x265 CRF 8 (High Quality)", 'libx265_8'), ("x264 Level 10 (High Quality)", 'libx264_10'), ("x264 Lossless", 'libx264_lossless')], value=self.server_config.get("video_output_codec", "libx264_8"), label="Video Codec")self.image_output_codec_choice = gr.Dropdown(choices=[("JPEG Q85", 'jpeg_85'), ("WEBP Q85", 'webp_85'), ("JPEG Q95", 'jpeg_95'), ("WEBP Q95", 'webp_95'), ("WEBP Lossless", 'webp_lossless'), ("PNG Lossless", 'png')], value=self.server_config.get("image_output_codec", "jpeg_95"), label="Image Codec")- self.audio_output_codec_choice = gr.Dropdown(choices=[("AAC 128 kbit", 'aac_128')], value=self.server_config.get("audio_output_codec", "aac_128"), visible=True, label="Audio Codec to use for mp4 container")+ self.audio_output_codec_choice = gr.Dropdown(+ choices=[+ ("AAC 128 kbps", "aac_128"),+ ("AAC 192 kbps", "aac_192"),+ ("AAC 256 kbps (High Quality, Recommended)", "aac_256"),+ ("AAC 320 kbps (Very High Quality)", "aac_320"),+ ("ALAC Lossless (preview/playback compatibility may be limited)", "alac"),+ ],+ value=self.server_config.get("audio_output_codec", "aac_128"),+ visible=True,+ label="Audio Codec to use for mp4 container",+ )audio_standalone_default = self.server_config.get("audio_stand_alone_output_codec", "wav")if audio_standalone_default == "mp3":audio_standalone_default = "mp3_192"@@ -296,9 +328,9 @@ class ConfigTabPlugin(WAN2GPPlugin):self.depth_anything_v2_variant_choice, self.vae_config_choice, self.boost_choice, self.enable_int8_kernels_choice,self.video_profile_choice, self.image_profile_choice, self.audio_profile_choice,self.preload_in_VRAM_choice, self.max_reserved_loras_choice,- self.enhancer_enabled_choice, self.enhancer_mode_choice,+ self.enhancer_enabled_choice, self.enhancer_quantization_choice, self.enhancer_mode_choice,self.prompt_enhancer_temperature_choice, self.prompt_enhancer_top_p_choice, self.prompt_enhancer_randomize_seed_choice,- self.mmaudio_mode_choice, self.mmaudio_persistence_choice, self.rife_version_choice,+ self.mmaudio_mode_choice, self.mmaudio_persistence_choice, self.rife_version_choice, self.matanyone_version_choice,self.video_output_codec_choice, self.image_output_codec_choice, self.audio_output_codec_choice, self.audio_stand_alone_output_codec_choice,self.metadata_choice, self.embed_source_images_choice,self.video_save_path_choice, self.image_save_path_choice, self.audio_save_path_choice,@@ -349,9 +381,9 @@ class ConfigTabPlugin(WAN2GPPlugin):depth_anything_v2_variant_choice, vae_config_choice, boost_choice, enable_int8_kernels_choice,video_profile_choice, image_profile_choice, audio_profile_choice,preload_in_VRAM_choice, max_reserved_loras_choice,- enhancer_enabled_choice, enhancer_mode_choice,+ enhancer_enabled_choice, enhancer_quantization_choice, enhancer_mode_choice,prompt_enhancer_temperature_choice, prompt_enhancer_top_p_choice, prompt_enhancer_randomize_seed_choice,- mmaudio_mode_choice, mmaudio_persistence_choice, rife_version_choice,+ mmaudio_mode_choice, mmaudio_persistence_choice, rife_version_choice, matanyone_version_choice,video_output_codec_choice, image_output_codec_choice, audio_output_codec_choice, audio_stand_alone_output_codec_choice,metadata_choice, embed_source_images_choice,save_path_choice, image_save_path_choice, audio_save_path_choice,@@ -381,9 +413,10 @@ class ConfigTabPlugin(WAN2GPPlugin):"boost": boost_choice, "enable_int8_kernels": enable_int8_kernels_choice, "clear_file_list": clear_file_list_choice,"preload_model_policy": preload_model_policy_choice, "UI_theme": UI_theme_choice,"fit_canvas": fit_canvas_choice, "enhancer_enabled": enhancer_enabled_choice,+ "prompt_enhancer_quantization": enhancer_quantization_choice,"enhancer_mode": enhancer_mode_choice, "mmaudio_mode": mmaudio_mode_choice,"mmaudio_persistence": mmaudio_persistence_choice, "mmaudio_enabled": mmaudio_enabled_choice,- "rife_version": rife_version_choice,+ "rife_version": rife_version_choice, "matanyone_version": matanyone_version_choice,"prompt_enhancer_temperature": prompt_enhancer_temperature_choice,"prompt_enhancer_top_p": prompt_enhancer_top_p_choice,"prompt_enhancer_randomize_seed": prompt_enhancer_randomize_seed_choice,@@ -426,8 +459,8 @@ class ConfigTabPlugin(WAN2GPPlugin):"attention_mode", "vae_config", "boost", "enable_int8_kernels", "save_path", "image_save_path", "audio_save_path","metadata_type", "clear_file_list", "fit_canvas", "depth_anything_v2_variant","notification_sound_enabled", "notification_sound_volume", "mmaudio_mode",- "mmaudio_persistence", "mmaudio_enabled", "rife_version",- "prompt_enhancer_temperature", "prompt_enhancer_top_p", "prompt_enhancer_randomize_seed",+ "mmaudio_persistence", "mmaudio_enabled", "rife_version", "matanyone_version",+ "prompt_enhancer_temperature", "prompt_enhancer_top_p", "prompt_enhancer_randomize_seed", "prompt_enhancer_quantization","max_frames_multiplier", "display_stats", "enable_4k_resolutions", "max_reserved_loras", "video_output_codec", "video_container","embed_source_images", "image_output_codec", "audio_output_codec", "audio_stand_alone_output_codec", "checkpoints_paths", "loras_root", "save_queue_if_crash","model_hierarchy_type", "UI_theme", "queue_color_scheme"@@ -458,7 +491,7 @@ class ConfigTabPlugin(WAN2GPPlugin):self.set_global("reload_needed", needs_reload)self.server_config.update(new_server_config)- if "enhancer_enabled" in changes or "enhancer_mode" in changes:+ if "enhancer_enabled" in changes or "enhancer_mode" in changes or "prompt_enhancer_quantization" in changes or "lm_decoder_engine" in changes:self.reset_prompt_enhancer()if "enable_int8_kernels" in changes:self.apply_int8_kernel_setting(new_server_config["enable_int8_kernels"], True)diff --git a/plugins/wan2gp-video-mask-creator/plugin.py b/plugins/wan2gp-video-mask-creator/plugin.pyindex 6ae3eab..ae80fb9 100644--- a/plugins/wan2gp-video-mask-creator/plugin.py+++ b/plugins/wan2gp-video-mask-creator/plugin.py@@ -14,8 +14,6 @@ class VideoMaskCreatorPlugin(WAN2GPPlugin):self.vmc_event_handler = self.matanyone_app.get_vmc_event_handler()def setup_ui(self):- self.request_global("download_shared_done")- self.request_global("download_models")self.request_global("server_config")self.request_global("get_current_model_settings")@@ -43,8 +41,7 @@ class VideoMaskCreatorPlugin(WAN2GPPlugin):def on_tab_select(self, state: dict) -> None:# print("[VideoMaskCreatorPlugin] Tab selected. Loading models...")- if not self.download_shared_done:- self.download_models()+ self.matanyone_app.ensure_selected_assets(self.server_config)self.vmc_event_handler(state, True)self._is_active = Truediff --git a/preprocessing/matanyone/app.py b/preprocessing/matanyone/app.pyindex 4c621b3..80da293 100644--- a/preprocessing/matanyone/app.py+++ b/preprocessing/matanyone/app.py@@ -16,9 +16,8 @@ from datetime import datetimefrom .tools.painter import mask_painterfrom .tools.interact_tools import SamControlerfrom .tools.misc import get_device-from .tools.download_util import load_file_from_urlfrom .tools.base_segmenter import set_image_encoder_patch-from .utils.get_default_model import get_matanyone_model+from .utils.model_assets import ensure_selected_matanyone_assets, get_matanyone_title_html, get_selected_matanyone_version, load_selected_matanyone_modelfrom .matanyone.inference.inference_core import InferenceCorefrom .matanyone_wrapper import matanyonefrom shared.utils.audio_video import save_video, save_image@@ -37,6 +36,8 @@ model_in_GPU = Falsematanyone_in_GPU = Falsebfloat16_supported = FalsePlugIn = None+server_config_ref = None+loaded_matanyone_version = None# SAM generatorimport copy@@ -688,7 +689,7 @@ def restart():def select_matanyone(state):global matanyone_in_GPU, model_in_GPU- if matanyone_model is None:+ if matanyone_model is None or loaded_matanyone_version != get_selected_matanyone_version(server_config_ref):load_unload_models(state, True, True)if matanyone_in_GPU: returnmodel.samcontroler.sam_controler.model.to("cpu")@@ -699,7 +700,7 @@ def select_matanyone(state):def select_SAM(state):global matanyone_in_GPU, model_in_GPU- if matanyone_model is None:+ if matanyone_model is None or loaded_matanyone_version != get_selected_matanyone_version(server_config_ref):load_unload_models(state, True, True)if model_in_GPU: returnmatanyone_model.to("cpu")@@ -713,9 +714,13 @@ load_in_progress = Falsedef load_unload_models(state = None, selected = True, force = False):global model_loaded, load_in_progressglobal model- global matanyone_model, matanyone_processor, matanyone_in_GPU , model_in_GPU, bfloat16_supported+ global matanyone_model, matanyone_processor, matanyone_in_GPU , model_in_GPU, bfloat16_supported, loaded_matanyone_versionif selected:+ selected_version = get_selected_matanyone_version(server_config_ref)+ if model_loaded and loaded_matanyone_version != selected_version:+ load_unload_models(state, False, True)+if (not force) and any_GPU_process_running(state, "matanyone"):return@@ -725,8 +730,7 @@ def load_unload_models(state = None, selected = True, force = False):return# print("Matanyone Tab Selected")if model_loaded or load_in_progress:- pass- # load_sam()+ returnelse:load_in_progress = True# args, defined in track_anything.py@@ -740,6 +744,7 @@ def load_unload_models(state = None, selected = True, force = False):# sam_checkpoint = load_file_from_url(sam_checkpoint_url_dict[arg_sam_model_type], ".")sam_checkpoint = None+ ensure_selected_matanyone_assets(server_config_ref)transfer_stream = torch.cuda.Stream()with torch.cuda.stream(transfer_stream):@@ -753,9 +758,7 @@ def load_unload_models(state = None, selected = True, force = False):model = MaskGenerator(sam_checkpoint, "cpu")model.samcontroler.sam_controler.model.to("cpu").to(torch.bfloat16).to(arg_device)model_in_GPU = True- from .matanyone.model.matanyone import MatAnyone- # matanyone_model = MatAnyone.from_pretrained("PeiqingYang/MatAnyone")- matanyone_model = MatAnyone.from_pretrained(fl.locate_folder("mask"))+ matanyone_model, loaded_matanyone_version, _ = load_selected_matanyone_model(server_config_ref)# pipe ={"mat" : matanyone_model, "sam" :model.samcontroler.sam_controler.model }# offload.profile(pipe)matanyone_model = matanyone_model.to("cpu").eval()@@ -770,6 +773,7 @@ def load_unload_models(state = None, selected = True, force = False):# model.samcontroler.sam_controler.model.to("cpu")# matanyone_model.to("cpu")model = matanyone_model = matanyone_processor = None+ loaded_matanyone_version = Nonematanyone_in_GPU = model_in_GPU = Falsegc.collect()torch.cuda.empty_cache()@@ -780,6 +784,14 @@ def get_vmc_event_handler():return load_unload_models+def ensure_selected_assets(server_config=None):+ return ensure_selected_matanyone_assets(server_config if server_config is not None else server_config_ref)+++def get_title_markdown():+ return get_matanyone_title_html(server_config_ref)++def export_image(state, image_output):ui_settings = get_current_model_settings(state)image_refs = ui_settings.get("image_refs", None)@@ -814,8 +826,9 @@ def teleport_to_video_tab(tab_state, state):def display(tabs, tab_state, state, refresh_form_trigger, server_config, get_current_model_settings_fn): #, vace_video_input, vace_image_input, vace_video_mask, vace_image_mask, vace_image_refs):# my_tab.select(fn=load_unload_models, inputs=[], outputs=[])- global image_output_codec, video_output_codec, get_current_model_settings+ global image_output_codec, video_output_codec, get_current_model_settings, server_config_refget_current_model_settings = get_current_model_settings_fn+ server_config_ref = server_configimage_output_codec = server_config.get("image_output_codec", None)video_output_codec = server_config.get("video_output_codec", None)@@ -837,7 +850,8 @@ def display(tabs, tab_state, state, refresh_form_trigger, server_config, get_cur# download assets- gr.Markdown("<B>Mask Edition is provided by MatAnyone v2, VRAM optimizations & Extended Masks by DeepBeepMeep</B>")+ matanyone_title_md = gr.Markdown(get_title_markdown())+ refresh_form_trigger.change(fn=get_title_markdown, inputs=[], outputs=[matanyone_title_md], show_progress="hidden")gr.Markdown("If you have some trouble creating the perfect mask, be aware of these tips:")gr.Markdown("- Using the Matanyone Settings you can also define Negative Point Prompts to remove parts of the current selection.")gr.Markdown("- Sometime it is very hard to fit everything you want in a single mask, it may be much easier to combine multiple independent sub Masks before producing the Matting : each sub Mask is created by selecting an area of an image and by clicking the Add Mask button. Sub masks can then be enabled / disabled in the Matanyone settings.")diff --git a/preprocessing/matanyone/utils/model_assets.py b/preprocessing/matanyone/utils/model_assets.pynew file mode 100644index 0000000..7d0f751--- /dev/null+++ b/preprocessing/matanyone/utils/model_assets.py@@ -0,0 +1,179 @@+import json+import os+import sys+from typing import Any++from huggingface_hub import hf_hub_download+from mmgp import offload+from omegaconf import OmegaConf++from shared.utils import files_locator as fl++from .model_signature import MATANYONE_V1, MATANYONE_V2, detect_matanyone_model_version++MATANYONE_SETTINGS_KEY = "matanyone_version"+MATANYONE_DEFAULT_VERSION = MATANYONE_V1+MATANYONE_REPO_ID = "DeepBeepMeep/Wan2.1"+MATANYONE_FOLDER = "mask"+MATANYONE_CONFIG_NAME = "config.json"+MATANYONE_SAM_NAME = "sam_vit_h_4b8939_fp16.safetensors"+MATANYONE_LEGACY_NAME = "model.safetensors"+MATANYONE_WEIGHT_FILES = {+ MATANYONE_V1: "matanyone.safetensors",+ MATANYONE_V2: "matanyone2.safetensors",+}+MATANYONE_VERSION_LABELS = {+ MATANYONE_V1: "MatAnyone v1 (original)",+ MATANYONE_V2: "MatAnyone v2",+}+++def _mask_relpath(filename: str) -> str:+ return os.path.join(MATANYONE_FOLDER, filename)+++def normalize_matanyone_version(value: Any) -> str:+ if value is None:+ return MATANYONE_DEFAULT_VERSION+ text = str(value).strip().lower()+ if text in {"2", MATANYONE_V2, "matanyone2"}:+ return MATANYONE_V2+ return MATANYONE_V1+++def _get_runtime_server_config(server_config=None):+ if isinstance(server_config, dict):+ return server_config+ main_module = sys.modules.get("__main__")+ runtime_config = getattr(main_module, "server_config", None)+ return runtime_config if isinstance(runtime_config, dict) else {}+++def _get_runtime_server_config_filename() -> str | None:+ main_module = sys.modules.get("__main__")+ filename = getattr(main_module, "server_config_filename", None)+ return filename if isinstance(filename, str) and len(filename) > 0 else None+++def _save_runtime_server_config(server_config) -> bool:+ filename = _get_runtime_server_config_filename()+ if not isinstance(server_config, dict) or filename is None:+ return False+ with open(filename, "w", encoding="utf-8") as writer:+ writer.write(json.dumps(server_config, indent=4))+ return True+++def get_selected_matanyone_version(server_config=None) -> str:+ runtime_config = _get_runtime_server_config(server_config)+ return normalize_matanyone_version(runtime_config.get(MATANYONE_SETTINGS_KEY, MATANYONE_DEFAULT_VERSION))+++def get_selected_matanyone_label(server_config=None) -> str:+ return MATANYONE_VERSION_LABELS[get_selected_matanyone_version(server_config)]+++def get_matanyone_title_html(server_config=None) -> str:+ return f"<B>Mask Edition is provided by {get_selected_matanyone_label(server_config)}, VRAM optimizations & Extended Masks by DeepBeepMeep</B>"+++def get_selected_matanyone_weight_name(server_config=None) -> str:+ return MATANYONE_WEIGHT_FILES[get_selected_matanyone_version(server_config)]+++def get_selected_matanyone_weights_path(server_config=None):+ selected_name = get_selected_matanyone_weight_name(server_config)+ selected_path = fl.locate_file(_mask_relpath(selected_name), error_if_none=False)+ if selected_path is not None:+ return selected_path++ legacy_path = fl.locate_file(_mask_relpath(MATANYONE_LEGACY_NAME), error_if_none=False)+ if legacy_path is None:+ return None++ if detect_matanyone_model_version(legacy_path) == get_selected_matanyone_version(server_config):+ return legacy_path+ return None+++def _download_mask_asset(filename: str) -> str:+ return hf_hub_download(repo_id=MATANYONE_REPO_ID, filename=filename, local_dir=fl.get_download_location(), subfolder=MATANYONE_FOLDER)+++def query_matanyone_download_def(server_config=None):+ runtime_config = _get_runtime_server_config(server_config)+ return {+ "repoId": MATANYONE_REPO_ID,+ "sourceFolderList": [MATANYONE_FOLDER],+ "fileList": [[MATANYONE_SAM_NAME, get_selected_matanyone_weight_name(runtime_config), MATANYONE_CONFIG_NAME]],+ }+++def migrate_matanyone_install(server_config=None):+ runtime_config = _get_runtime_server_config(server_config)+ runtime_config.setdefault(MATANYONE_SETTINGS_KEY, MATANYONE_DEFAULT_VERSION)++ legacy_path = fl.locate_file(_mask_relpath(MATANYONE_LEGACY_NAME), error_if_none=False)+ if legacy_path is None:+ return None++ legacy_version = detect_matanyone_model_version(legacy_path)+ if legacy_version is None:+ return None++ target_name = MATANYONE_WEIGHT_FILES[legacy_version]+ target_path = fl.get_download_location(_mask_relpath(target_name))+ os.makedirs(os.path.dirname(target_path), exist_ok=True)++ if os.path.normcase(os.path.abspath(legacy_path)) != os.path.normcase(os.path.abspath(target_path)):+ if os.path.isfile(target_path):+ os.remove(legacy_path)+ else:+ os.replace(legacy_path, target_path)++ config_changed = False+ if legacy_version == MATANYONE_V2 and runtime_config.get(MATANYONE_SETTINGS_KEY) != MATANYONE_V2:+ runtime_config[MATANYONE_SETTINGS_KEY] = MATANYONE_V2+ config_changed = True+ elif MATANYONE_SETTINGS_KEY not in runtime_config:+ runtime_config[MATANYONE_SETTINGS_KEY] = legacy_version+ config_changed = True++ if config_changed:+ _save_runtime_server_config(runtime_config)++ if legacy_version == MATANYONE_V2:+ return "Migrated legacy MatAnyone v2 weights to 'mask/matanyone2.safetensors' and selected MatAnyone v2."+ return "Migrated legacy MatAnyone v1 weights to 'mask/matanyone.safetensors'."+++def ensure_selected_matanyone_assets(server_config=None):+ runtime_config = _get_runtime_server_config(server_config)+ migrate_matanyone_install(runtime_config)++ for filename in query_matanyone_download_def(runtime_config)["fileList"][0]:+ if filename == get_selected_matanyone_weight_name(runtime_config):+ continue+ if fl.locate_file(_mask_relpath(filename), error_if_none=False) is None:+ _download_mask_asset(filename)++ weights_path = get_selected_matanyone_weights_path(runtime_config)+ if weights_path is None:+ weights_path = _download_mask_asset(get_selected_matanyone_weight_name(runtime_config))++ config_path = fl.locate_file(_mask_relpath(MATANYONE_CONFIG_NAME))+ sam_path = fl.locate_file(_mask_relpath(MATANYONE_SAM_NAME))+ return config_path, weights_path, sam_path+++def load_selected_matanyone_model(server_config=None):+ config_path, weights_path, _ = ensure_selected_matanyone_assets(server_config)++ with open(config_path, "r", encoding="utf-8-sig") as reader:+ config_data = json.load(reader)++ from ..matanyone.model.matanyone import MatAnyone++ model = MatAnyone(OmegaConf.create(config_data["cfg"]), single_object=config_data.get("single_object", True)).eval()+ offload.load_model_data(model, weights_path, writable_tensors=False)+ return model, get_selected_matanyone_version(server_config), weights_pathdiff --git a/preprocessing/matanyone/utils/model_signature.py b/preprocessing/matanyone/utils/model_signature.pyindex 4ba5565..044291a 100644--- a/preprocessing/matanyone/utils/model_signature.py+++ b/preprocessing/matanyone/utils/model_signature.py@@ -1,19 +1,30 @@import os+MATANYONE_V1 = "v1"+MATANYONE_V2 = "v2"MATANYONE2_SIGNATURE_OFFSET = 61576+MATANYONE1_SIGNATURE_BYTES = bytes([23, 65, 57, 61, 52, 154, 201, 60, 231, 125, 180, 188, 187, 85, 166, 61])MATANYONE2_SIGNATURE_BYTES = bytes([155, 244, 100, 61, 100, 179, 194, 60, 92, 239, 124, 186, 254, 127, 168, 61])-def delete_if_not_matanyone2_model(model_path):+def detect_matanyone_model_version(model_path):if model_path is None or not os.path.isfile(model_path):- return False+ return Nonewith open(model_path, "rb") as reader:reader.seek(MATANYONE2_SIGNATURE_OFFSET)actual = reader.read(len(MATANYONE2_SIGNATURE_BYTES))+ if actual == MATANYONE1_SIGNATURE_BYTES:+ return MATANYONE_V1if actual == MATANYONE2_SIGNATURE_BYTES:+ return MATANYONE_V2+ return None+++def delete_if_not_matanyone2_model(model_path):+ if detect_matanyone_model_version(model_path) is not None:return Falseos.remove(model_path)diff --git a/requirements.txt b/requirements.txtindex 7a12206..e857531 100644--- a/requirements.txt+++ b/requirements.txt@@ -73,6 +73,7 @@ peft==0.17.0 # 0.15.0vector-quantize-pytorchmatplotlibgguf==0.17.1+flash-linear-attention==0.4.1# Utilitiesftfydiff --git a/shared/api.py b/shared/api.pynew file mode 100644index 0000000..9bb0bc8--- /dev/null+++ b/shared/api.py@@ -0,0 +1,983 @@+"""Lightweight in-process API wrapper around WanGP generation."""++from __future__ import annotations++import contextlib+import copy+import importlib+import inspect+import io+import json+import os+import queue+import sys+import threading+import time+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Iterator, Sequence++from PIL import Image++from shared.utils.thread_utils import AsyncStream++_RUNTIME_LOCK = threading.RLock()+_GENERATION_LOCK = threading.RLock()+_RUNTIME: "_WanGPRuntime | None" = None+_BANNER_PRINTED = False+++@dataclass(frozen=True)+class StreamMessage:+ stream: str+ text: str+++@dataclass(frozen=True)+class ProgressUpdate:+ phase: str+ status: str+ progress: int+ current_step: int | None+ total_steps: int | None+ raw_phase: str | None = None+ unit: str | None = None+++@dataclass(frozen=True)+class PreviewUpdate:+ image: Image.Image | None+ phase: str+ status: str+ progress: int+ current_step: int | None+ total_steps: int | None+++@dataclass(frozen=True)+class SessionEvent:+ kind: str+ data: Any = None+ timestamp: float = field(default_factory=time.time)+++@dataclass(frozen=True)+class GenerationResult:+ success: bool+ generated_files: list[str]+ errors: list["GenerationError"]+ total_tasks: int+ successful_tasks: int+ failed_tasks: int+++@dataclass(frozen=True)+class GenerationError:+ message: str+ task_index: int | None = None+ task_id: Any = None+ stage: str | None = None++ def __str__(self) -> str:+ return self.message+++class SessionStream:+ def __init__(self) -> None:+ self._queue: queue.Queue[SessionEvent | object] = queue.Queue()+ self._closed = threading.Event()+ self._sentinel = object()++ def put(self, kind: str, data: Any = None) -> None:+ if self._closed.is_set():+ return+ self._queue.put(SessionEvent(kind=kind, data=data))++ def close(self) -> None:+ if self._closed.is_set():+ return+ self._closed.set()+ self._queue.put(self._sentinel)++ def get(self, timeout: float | None = None) -> SessionEvent | None:+ try:+ item = self._queue.get(timeout=timeout)+ except queue.Empty:+ return None+ if item is self._sentinel:+ return None+ return item++ def iter(self, timeout: float | None = None) -> Iterator[SessionEvent]:+ while True:+ event = self.get(timeout=timeout)+ if event is None:+ if self._closed.is_set():+ break+ continue+ yield event++ @property+ def closed(self) -> bool:+ return self._closed.is_set()+++class _OutputCapture(io.TextIOBase):+ def __init__(self, stream_name: str, emit_line, console: io.TextIOBase | None = None) -> None:+ self._stream_name = stream_name+ self._emit_line = emit_line+ self._console = console+ self._buffer = ""++ def writable(self) -> bool:+ return True++ def write(self, text: str) -> int:+ if not text:+ return 0+ if self._console is not None:+ self._console.write(text)+ self._buffer += text+ self._drain(False)+ return len(text)++ def flush(self) -> None:+ if self._console is not None:+ self._console.flush()+ self._drain(True)++ def _drain(self, flush_all: bool) -> None:+ while True:+ split_at = -1+ for delimiter in ("\r", "\n"):+ index = self._buffer.find(delimiter)+ if index >= 0 and (split_at < 0 or index < split_at):+ split_at = index+ if split_at < 0:+ break+ line = self._buffer[:split_at]+ self._buffer = self._buffer[split_at + 1 :]+ if line:+ self._emit_line(self._stream_name, line)+ if flush_all and self._buffer:+ self._emit_line(self._stream_name, self._buffer)+ self._buffer = ""+++@dataclass(frozen=True)+class _WanGPRuntime:+ module: Any+ root: Path+ config_path: Path+ cli_args: tuple[str, ...]+++class SessionJob:+ def __init__(self, session: "WanGPSession") -> None:+ self._session = session+ self.events = SessionStream()+ self._done = threading.Event()+ self._cancel_requested = threading.Event()+ self._thread: threading.Thread | None = None+ self._result: GenerationResult | None = None++ def _bind_thread(self, thread: threading.Thread) -> None:+ self._thread = thread++ def _set_result(self, result: GenerationResult) -> None:+ self._result = result+ self._done.set()++ def cancel(self) -> None:+ self._cancel_requested.set()++ def result(self, timeout: float | None = None) -> GenerationResult:+ if not self._done.wait(timeout=timeout):+ raise TimeoutError("WanGP session job timed out")+ return self._result or GenerationResult(+ success=False,+ generated_files=[],+ errors=[],+ total_tasks=0,+ successful_tasks=0,+ failed_tasks=0,+ )++ def join(self, timeout: float | None = None) -> GenerationResult:+ return self.result(timeout=timeout)++ @property+ def done(self) -> bool:+ return self._done.is_set()++ @property+ def cancel_requested(self) -> bool:+ return self._cancel_requested.is_set()+++class WanGPSession:+ def __init__(+ self,+ *,+ root: str | os.PathLike[str] | None = None,+ config_path: str | os.PathLike[str] | None = None,+ output_dir: str | os.PathLike[str] | None = None,+ callbacks: object | None = None,+ cli_args: Sequence[str] = (),+ console_output: bool = True,+ ) -> None:+ self._root = Path(root or Path(__file__).resolve().parents[1]).resolve()+ self._config_path = Path(config_path).resolve() if config_path is not None else (self._root / "wgp_config.json").resolve()+ self._output_dir = Path(output_dir).resolve() if output_dir is not None else None+ self._callbacks = callbacks+ self._cli_args = tuple(str(arg) for arg in cli_args)+ self._console_output = bool(console_output)+ self._state = self._create_headless_state()+ self._active_job: SessionJob | None = None+ self._job_lock = threading.Lock()+ self._attachment_keys: tuple[str, ...] | None = None++ def ensure_ready(self) -> "WanGPSession":+ self._ensure_runtime()+ return self++ def submit(self, source: str | os.PathLike[str] | dict[str, Any] | list[dict[str, Any]]) -> SessionJob:+ tasks = self._normalize_source(source, caller_base_path=self._get_caller_base_path())+ return self._submit_tasks(tasks)++ def submit_task(self, settings: dict[str, Any]) -> SessionJob:+ caller_base_path = self._get_caller_base_path()+ task = self._normalize_task(settings, task_index=1)+ return self._submit_tasks([self._absolutize_task_paths(task, caller_base_path)])++ def submit_manifest(self, settings_list: list[dict[str, Any]]) -> SessionJob:+ caller_base_path = self._get_caller_base_path()+ tasks = [+ self._absolutize_task_paths(self._normalize_task(settings, task_index=index + 1), caller_base_path)+ for index, settings in enumerate(settings_list)+ ]+ return self._submit_tasks(tasks)++ def run(self, source: str | os.PathLike[str] | dict[str, Any] | list[dict[str, Any]]) -> GenerationResult:+ return self.submit(source).result()++ def run_task(self, settings: dict[str, Any]) -> GenerationResult:+ return self.submit_task(settings).result()++ def run_manifest(self, settings_list: list[dict[str, Any]]) -> GenerationResult:+ return self.submit_manifest(settings_list).result()++ def close(self) -> None:+ runtime = self._ensure_runtime()+ with _GENERATION_LOCK, _pushd(runtime.root):+ runtime.module.release_model()++ def cancel(self) -> None:+ with self._job_lock:+ job = self._active_job+ if job is not None:+ job.cancel()++ @staticmethod+ def _create_headless_state() -> dict[str, Any]:+ return {+ "gen": {+ "queue": [],+ "in_progress": False,+ "file_list": [],+ "file_settings_list": [],+ "audio_file_list": [],+ "audio_file_settings_list": [],+ "selected": 0,+ "audio_selected": 0,+ "prompt_no": 0,+ "prompts_max": 0,+ "repeat_no": 0,+ "total_generation": 1,+ "window_no": 0,+ "total_windows": 0,+ "progress_status": "",+ "process_status": "process:main",+ },+ "loras": [],+ }++ def _submit_tasks(self, tasks: list[dict[str, Any]]) -> SessionJob:+ with self._job_lock:+ if self._active_job is not None and not self._active_job.done:+ raise RuntimeError("WanGP session already has a generation in progress")+ job = SessionJob(self)+ thread = threading.Thread(+ target=self._run_job,+ args=(job, copy.deepcopy(tasks)),+ daemon=True,+ name="wangp-session-job",+ )+ job._bind_thread(thread)+ self._active_job = job+ thread.start()+ return job++ def _run_job(self, job: SessionJob, tasks: list[dict[str, Any]]) -> None:+ stream = AsyncStream()+ gen = self._state["gen"]+ worker_done = threading.Event()+ base_file_count = len(gen["file_list"])+ base_audio_count = len(gen["audio_file_list"])+ total_tasks = len(tasks)+ runtime: _WanGPRuntime | None = None+ task_summary: dict[str, Any] = {+ "errors": [],+ "successful_tasks": 0,+ "failed_tasks": 0,+ "total_tasks": total_tasks,+ }++ try:+ runtime = self._ensure_runtime()+ with _GENERATION_LOCK, _pushd(runtime.root):+ self._configure_runtime(runtime)+ self._prepare_state_for_run(tasks)+ job.events.put("started", {"tasks": len(tasks)})++ def worker() -> None:+ stdout_capture = _OutputCapture(+ "stdout",+ lambda stream_name, line: self._emit_stream(job, stream_name, line),+ console=sys.__stdout__ if self._console_output else None,+ )+ stderr_capture = _OutputCapture(+ "stderr",+ lambda stream_name, line: self._emit_stream(job, stream_name, line),+ console=sys.__stderr__ if self._console_output else None,+ )+ try:+ with contextlib.redirect_stdout(stdout_capture), contextlib.redirect_stderr(stderr_capture):+ self._run_tasks_worker(runtime.module, tasks, stream, job, task_summary)+ except BaseException as exc:+ failure = self._make_generation_error(+ exc,+ task_index=None,+ task_id=None,+ stage="runtime",+ )+ task_summary["errors"].append(failure)+ stream.output_queue.push("error", failure)+ finally:+ stdout_capture.flush()+ stderr_capture.flush()+ stream.output_queue.push("worker_exit", None)+ worker_done.set()++ worker_thread = threading.Thread(target=worker, daemon=True, name="wangp-session-worker")+ worker_thread.start()++ while True:+ if job.cancel_requested:+ self._request_cancel_unlocked(runtime.module)+ item = stream.output_queue.pop()+ if item is None:+ if worker_done.is_set() and not worker_thread.is_alive():+ break+ time.sleep(0.01)+ continue+ command, data = item+ if command == "worker_exit":+ break+ self._handle_command(job, runtime.module, tasks, command, data)++ worker_thread.join(timeout=0.1)+ outputs = self._collect_outputs(base_file_count, base_audio_count)+ if job.cancel_requested and not task_summary["errors"]:+ task_summary["errors"].append(+ GenerationError(message="Generation was cancelled", stage="cancelled")+ )+ task_summary["failed_tasks"] = max(task_summary["failed_tasks"], 1)+ result = GenerationResult(+ success=not task_summary["errors"],+ generated_files=outputs,+ errors=list(task_summary["errors"]),+ total_tasks=task_summary["total_tasks"],+ successful_tasks=task_summary["successful_tasks"],+ failed_tasks=task_summary["failed_tasks"],+ )+ job.events.put("completed", result)+ self._emit_callback("on_complete", result)+ job._set_result(result)+ except BaseException as exc:+ failure = self._make_generation_error(exc, task_index=None, task_id=None, stage="runtime")+ result = GenerationResult(+ success=False,+ generated_files=[],+ errors=[failure],+ total_tasks=total_tasks,+ successful_tasks=task_summary["successful_tasks"],+ failed_tasks=max(task_summary["failed_tasks"], 1 if total_tasks > 0 else 0),+ )+ job.events.put("error", failure)+ self._emit_callback("on_error", failure)+ job.events.put("completed", result)+ self._emit_callback("on_complete", result)+ job._set_result(result)+ finally:+ job.events.close()+ if runtime is not None:+ self._reset_state_after_run()+ with self._job_lock:+ if self._active_job is job:+ self._active_job = None++ def _run_tasks_worker(+ self,+ wgp,+ tasks: list[dict[str, Any]],+ stream: AsyncStream,+ job: SessionJob,+ task_summary: dict[str, Any],+ ) -> None:+ expected_args = set(inspect.signature(wgp.generate_video).parameters.keys())+ total_tasks = len(tasks)++ for task_index, task in enumerate(tasks, start=1):+ if job.cancel_requested:+ break++ self._state["gen"]["prompt_no"] = task_index+ self._state["gen"]["prompts_max"] = total_tasks+ self._state["gen"]["queue"] = tasks+ task_id = task.get("id")+ task_errors: list[GenerationError] = []++ def send_cmd(command: str, data: Any = None) -> None:+ if command == "error":+ failure = self._make_generation_error(+ data,+ task_index=task_index,+ task_id=task_id,+ stage="generation",+ )+ task_errors.append(failure)+ stream.output_queue.push("error", failure)+ return+ stream.output_queue.push(command, data)++ validated_settings = wgp.validate_task(task, self._state)+ if validated_settings is None:+ failure = GenerationError(+ message=f"Task {task_index} failed validation",+ task_index=task_index,+ task_id=task_id,+ stage="validation",+ )+ task_summary["errors"].append(failure)+ task_summary["failed_tasks"] += 1+ stream.output_queue.push("error", failure)+ continue++ task_settings = validated_settings.copy()+ task_settings["state"] = self._state+ filtered_params = {key: value for key, value in task_settings.items() if key in expected_args}+ plugin_data = task.get("plugin_data", {})+ try:+ success = wgp.generate_video(task, send_cmd, plugin_data=plugin_data, **filtered_params)+ except BaseException as exc:+ if not task_errors:+ task_errors.append(+ self._make_generation_error(+ exc,+ task_index=task_index,+ task_id=task_id,+ stage="generation",+ )+ )+ stream.output_queue.push("error", task_errors[-1])+ success = False++ if self._state["gen"].get("abort", False) or job.cancel_requested:+ task_errors.append(+ GenerationError(+ message="Generation was cancelled",+ task_index=task_index,+ task_id=task_id,+ stage="cancelled",+ )+ )+ stream.output_queue.push("error", task_errors[-1])+ task_summary["errors"].extend(task_errors)+ task_summary["failed_tasks"] += 1+ break++ if task_errors:+ task_summary["errors"].extend(task_errors)+ task_summary["failed_tasks"] += 1+ continue++ if not success:+ failure = GenerationError(+ message=f"Task {task_index} did not complete successfully",+ task_index=task_index,+ task_id=task_id,+ stage="generation",+ )+ task_summary["errors"].append(failure)+ task_summary["failed_tasks"] += 1+ stream.output_queue.push("error", failure)+ continue++ task_summary["successful_tasks"] += 1++ def _handle_command(self, job: SessionJob, wgp, tasks: list[dict[str, Any]], command: str, data: Any) -> None:+ if command == "progress":+ progress = self._build_progress_update(data)+ job.events.put("progress", progress)+ self._emit_callback("on_progress", progress)+ return+ if command == "preview":+ preview = self._build_preview_update(wgp, tasks, data)+ if preview is not None:+ job.events.put("preview", preview)+ self._emit_callback("on_preview", preview)+ return+ if command == "status":+ text = str(data or "")+ job.events.put("status", text)+ self._emit_callback("on_status", text)+ return+ if command == "info":+ text = str(data or "")+ job.events.put("info", text)+ self._emit_callback("on_info", text)+ return+ if command == "output":+ job.events.put("output", data)+ self._emit_callback("on_output", data)+ return+ if command == "refresh_models":+ job.events.put("refresh_models", data)+ return+ if command == "error":+ error = data if isinstance(data, GenerationError) else self._make_generation_error(data)+ job.events.put("error", error)+ self._emit_callback("on_error", error)+ return+ job.events.put(command, data)++ def _build_progress_update(self, data: Any) -> ProgressUpdate:+ current_step: int | None = None+ total_steps: int | None = None+ status = ""+ unit: str | None = None++ if isinstance(data, list) and data:+ head = data[0]+ if isinstance(head, tuple) and len(head) == 2:+ current_step = int(head[0])+ total_steps = int(head[1])+ status = str(data[1] if len(data) > 1 else "")+ if len(data) > 3:+ unit = str(data[3])+ else:+ status = str(data[1] if len(data) > 1 else head)+ else:+ status = str(data or "")++ raw_phase = None+ progress_phase = self._state["gen"].get("progress_phase")+ if isinstance(progress_phase, tuple) and progress_phase:+ raw_phase = str(progress_phase[0] or "")+ phase = self._normalize_phase(raw_phase or status)+ progress = self._estimate_progress(phase, current_step, total_steps)+ return ProgressUpdate(+ phase=phase,+ status=status,+ progress=progress,+ current_step=current_step,+ total_steps=total_steps,+ raw_phase=raw_phase,+ unit=unit,+ )++ def _build_preview_update(self, wgp, tasks: list[dict[str, Any]], payload: Any) -> PreviewUpdate | None:+ progress = self._build_progress_update([0, self._state["gen"].get("progress_status", "")])+ model_type = ""+ queue_tasks = self._state["gen"].get("queue") or tasks+ if queue_tasks:+ model_type = str(self._get_task_settings(queue_tasks[0]).get("model_type", ""))+ image = wgp.generate_preview(model_type, payload) if model_type else None+ return PreviewUpdate(+ image=image,+ phase=progress.phase,+ status=progress.status,+ progress=progress.progress,+ current_step=progress.current_step,+ total_steps=progress.total_steps,+ )++ def _emit_stream(self, job: SessionJob, stream_name: str, line: str) -> None:+ message = StreamMessage(stream=stream_name, text=line)+ job.events.put("stream", message)+ self._emit_callback("on_stream", message)++ def _emit_callback(self, method_name: str, payload: Any) -> None:+ callback = self._callbacks+ if callback is None:+ return+ method = getattr(callback, method_name, None)+ if callable(method):+ method(payload)+ on_event = getattr(callback, "on_event", None)+ if callable(on_event):+ on_event(SessionEvent(kind=method_name.removeprefix("on_"), data=payload))++ def _configure_runtime(self, runtime: _WanGPRuntime) -> None:+ runtime.module.server_config["notification_sound_enabled"] = 0+ if self._output_dir is not None:+ self._output_dir.mkdir(parents=True, exist_ok=True)+ runtime.module.server_config["save_path"] = str(self._output_dir)+ runtime.module.server_config["image_save_path"] = str(self._output_dir)+ runtime.module.server_config["audio_save_path"] = str(self._output_dir)+ runtime.module.save_path = str(self._output_dir)+ runtime.module.image_save_path = str(self._output_dir)+ runtime.module.audio_save_path = str(self._output_dir)+ for output_path in (+ runtime.module.save_path,+ runtime.module.image_save_path,+ runtime.module.audio_save_path,+ ):+ Path(output_path).mkdir(parents=True, exist_ok=True)++ def _prepare_state_for_run(self, tasks: list[dict[str, Any]]) -> None:+ gen = self._state["gen"]+ gen["queue"] = tasks+ gen["process_status"] = "process:main"+ gen["progress_status"] = ""+ gen["progress_phase"] = ("", -1)+ gen["abort"] = False+ gen["early_stop"] = False+ gen["early_stop_forwarded"] = False+ gen["preview"] = None+ gen["status"] = "Generating..."+ gen["in_progress"] = True+ self._ensure_runtime().module.gen_in_progress = True++ def _reset_state_after_run(self) -> None:+ gen = self._state["gen"]+ gen["queue"] = []+ gen["process_status"] = "process:main"+ gen["progress_status"] = ""+ gen["progress_phase"] = ("", -1)+ gen["abort"] = False+ gen["early_stop"] = False+ gen["early_stop_forwarded"] = False+ gen.pop("in_progress", None)+ self._ensure_runtime().module.gen_in_progress = False++ def _collect_outputs(self, base_file_count: int, base_audio_count: int) -> list[str]:+ gen = self._state["gen"]+ files = gen["file_list"][base_file_count:]+ audio_files = gen["audio_file_list"][base_audio_count:]+ return [str(Path(path).resolve()) for path in [*files, *audio_files]]++ def _request_cancel_unlocked(self, wgp) -> None:+ gen = self._state["gen"]+ gen["resume"] = True+ gen["abort"] = True+ if wgp.wan_model is not None:+ wgp.wan_model._interrupt = True++ def _normalize_source(+ self,+ source: str | os.PathLike[str] | dict[str, Any] | list[dict[str, Any]],+ *,+ caller_base_path: Path,+ ) -> list[dict[str, Any]]:+ if isinstance(source, (str, os.PathLike)):+ return self._load_tasks_from_path(self._resolve_source_path(Path(source), caller_base_path), caller_base_path)+ if isinstance(source, list):+ return [+ self._absolutize_task_paths(self._normalize_task(task, task_index=index + 1), caller_base_path)+ for index, task in enumerate(source)+ ]+ if isinstance(source, dict):+ if isinstance(source.get("tasks"), list):+ tasks = source["tasks"]+ return [+ self._absolutize_task_paths(self._normalize_task(task, task_index=index + 1), caller_base_path)+ for index, task in enumerate(tasks)+ ]+ return [self._absolutize_task_paths(self._normalize_task(source, task_index=1), caller_base_path)]+ raise TypeError("WanGP session source must be a path, a settings dict, or a manifest list")++ def _normalize_task(self, task: dict[str, Any], *, task_index: int) -> dict[str, Any]:+ if not isinstance(task, dict):+ raise TypeError(f"Task {task_index} must be a dictionary")+ normalized = copy.deepcopy(task)+ if "settings" in normalized and "params" not in normalized:+ normalized["params"] = normalized.pop("settings")+ if "params" not in normalized:+ normalized = {"id": task_index, "params": normalized, "plugin_data": {}}+ normalized.setdefault("id", task_index)+ normalized.setdefault("plugin_data", {})+ normalized.setdefault("params", {})+ settings = normalized["params"]+ if isinstance(settings, dict):+ self._normalize_settings_values(settings)+ normalized.setdefault("prompt", settings.get("prompt", ""))+ normalized.setdefault("length", settings.get("video_length"))+ normalized.setdefault("steps", settings.get("num_inference_steps"))+ normalized.setdefault("repeats", settings.get("repeat_generation", 1))+ return normalized++ @staticmethod+ def _normalize_settings_values(settings: dict[str, Any]) -> None:+ force_fps = settings.get("force_fps")+ if isinstance(force_fps, (int, float)) and not isinstance(force_fps, bool):+ if isinstance(force_fps, float) and not force_fps.is_integer():+ settings["force_fps"] = str(force_fps)+ else:+ settings["force_fps"] = str(int(force_fps))++ @staticmethod+ def _get_task_settings(task: dict[str, Any]) -> dict[str, Any]:+ settings = task.get("params")+ if isinstance(settings, dict):+ return settings+ settings = task.get("settings")+ if isinstance(settings, dict):+ return settings+ return {}++ def _load_tasks_from_path(self, path: Path, caller_base_path: Path) -> list[dict[str, Any]]:+ runtime = self._ensure_runtime()+ if not path.exists():+ raise FileNotFoundError(path)+ if path.suffix.lower() == ".json":+ return self._load_settings_json(path, caller_base_path)+ with _pushd(runtime.root):+ tasks, error = runtime.module._parse_queue_zip(str(path), self._state)+ if error:+ raise RuntimeError(error)+ return [self._normalize_task(task, task_index=index + 1) for index, task in enumerate(tasks)]++ def _load_settings_json(self, path: Path, caller_base_path: Path) -> list[dict[str, Any]]:+ with path.open("r", encoding="utf-8") as handle:+ payload = json.load(handle)++ if isinstance(payload, list):+ raw_tasks = payload+ elif isinstance(payload, dict) and isinstance(payload.get("tasks"), list):+ raw_tasks = payload["tasks"]+ elif isinstance(payload, dict):+ raw_tasks = [payload]+ else:+ raise RuntimeError("Settings file must contain a JSON object or a list of tasks")++ tasks = [self._normalize_task(task, task_index=index + 1) for index, task in enumerate(raw_tasks)]+ return [self._absolutize_task_paths(task, caller_base_path) for task in tasks]++ @staticmethod+ def _get_caller_base_path() -> Path:+ return Path.cwd().resolve()++ @staticmethod+ def _resolve_source_path(path: Path, caller_base_path: Path) -> Path:+ if path.is_absolute():+ return path.resolve()+ return (caller_base_path / path).resolve()++ def _absolutize_task_paths(self, task: dict[str, Any], caller_base_path: Path) -> dict[str, Any]:+ normalized = copy.deepcopy(task)+ settings = normalized.get("params")+ if not isinstance(settings, dict):+ return normalized+ for key in self._get_attachment_keys():+ if key not in settings:+ continue+ settings[key] = self._absolutize_setting_path(settings[key], caller_base_path)+ return normalized++ def _get_attachment_keys(self) -> tuple[str, ...]:+ if self._attachment_keys is None:+ runtime = self._ensure_runtime()+ keys = getattr(runtime.module, "ATTACHMENT_KEYS", ())+ self._attachment_keys = tuple(str(key) for key in keys)+ return self._attachment_keys++ def _absolutize_setting_path(self, value: Any, caller_base_path: Path) -> Any:+ if isinstance(value, list):+ return [self._absolutize_setting_path(item, caller_base_path) for item in value]+ if isinstance(value, os.PathLike):+ value = os.fspath(value)+ if not isinstance(value, str) or not value.strip():+ return value+ path = Path(value)+ if path.is_absolute():+ return str(path.resolve())+ return str((caller_base_path / path).resolve())++ @staticmethod+ def _make_generation_error(+ error: Any,+ *,+ task_index: int | None = None,+ task_id: Any = None,+ stage: str | None = None,+ ) -> GenerationError:+ if isinstance(error, GenerationError):+ return error+ if isinstance(error, BaseException):+ message = str(error) or error.__class__.__name__+ else:+ message = str(error)+ return GenerationError(message=message, task_index=task_index, task_id=task_id, stage=stage)++ def _ensure_runtime(self) -> _WanGPRuntime:+ global _RUNTIME+ with _RUNTIME_LOCK:+ if _RUNTIME is not None:+ if _RUNTIME.root != self._root or _RUNTIME.config_path != self._config_path or _RUNTIME.cli_args != self._cli_args:+ raise RuntimeError("WanGP runtime already loaded with different root/config/cli args")+ return _RUNTIME++ argv = ["wgp.py", *self._cli_args]+ default_config_path = (self._root / "wgp_config.json").resolve()+ if self._config_path.name != "wgp_config.json":+ raise ValueError("config_path must point to a file named 'wgp_config.json'")+ if self._config_path != default_config_path:+ self._config_path.parent.mkdir(parents=True, exist_ok=True)+ if "--config" not in argv:+ argv.extend(["--config", str(self._config_path.parent)])++ if str(self._root) not in sys.path:+ sys.path.insert(0, str(self._root))++ with _pushd(self._root), _temporary_argv(argv):+ module = importlib.import_module("wgp")+ module_root = Path(module.__file__).resolve().parent+ if module_root != self._root:+ raise RuntimeError(f"WanGP module already loaded from {module_root}, expected {self._root}")+ if not hasattr(module, "app"):+ module.app = module.WAN2GPApplication()+ module.download_ffmpeg()++ _RUNTIME = _WanGPRuntime(+ module=module,+ root=self._root,+ config_path=self._config_path,+ cli_args=self._cli_args,+ )+ _print_banner_once(module)+ return _RUNTIME++ @staticmethod+ def _normalize_phase(text: str | None) -> str:+ lowered = str(text or "").lower()+ if "denoising first pass" in lowered or "denoising 1st pass" in lowered:+ return "inference_stage_1"+ if "denoising second pass" in lowered or "denoising 2nd pass" in lowered:+ return "inference_stage_2"+ if "denoising third pass" in lowered or "denoising 3rd pass" in lowered:+ return "inference_stage_3"+ if "loading model" in lowered or lowered.startswith("loading"):+ return "loading_model"+ if "enhancing prompt" in lowered or "encoding prompt" in lowered or "encoding" in lowered:+ return "encoding_text"+ if "vae decoding" in lowered or "decoding" in lowered:+ return "decoding"+ if "saved" in lowered or "completed" in lowered or "output" in lowered:+ return "downloading_output"+ if "cancel" in lowered or "abort" in lowered:+ return "cancelled"+ return "inference"++ @staticmethod+ def _estimate_progress(phase: str, current_step: int | None, total_steps: int | None) -> int:+ if total_steps is None or total_steps <= 0 or current_step is None:+ if phase == "loading_model":+ return 10+ if phase == "encoding_text":+ return 18+ if phase == "inference_stage_1":+ return 25+ if phase == "inference_stage_2":+ return 70+ if phase == "inference_stage_3":+ return 80+ if phase == "decoding":+ return 90+ if phase == "downloading_output":+ return 95+ if phase == "cancelled":+ return 0+ return 15+ ratio = max(0.0, min(1.0, current_step / total_steps))+ if phase == "loading_model":+ return min(15, 5 + int(ratio * 10))+ if phase == "encoding_text":+ return min(22, 12 + int(ratio * 10))+ if phase == "inference_stage_1":+ return min(68, 20 + int(ratio * 48))+ if phase == "inference_stage_2":+ return min(88, 68 + int(ratio * 20))+ if phase == "inference_stage_3":+ return min(89, 80 + int(ratio * 9))+ if phase == "decoding":+ return min(95, 85 + int(ratio * 10))+ if phase == "downloading_output":+ return min(98, 92 + int(ratio * 6))+ if phase == "cancelled":+ return 0+ return min(90, 20 + int(ratio * 65))+++def init(+ *,+ root: str | os.PathLike[str] | None = None,+ config_path: str | os.PathLike[str] | None = None,+ output_dir: str | os.PathLike[str] | None = None,+ callbacks: object | None = None,+ cli_args: Sequence[str] = (),+ console_output: bool = True,+) -> WanGPSession:+ """Create and eagerly initialize a reusable WanGP session."""++ return WanGPSession(+ root=root,+ config_path=config_path,+ output_dir=output_dir,+ callbacks=callbacks,+ cli_args=cli_args,+ console_output=console_output,+ ).ensure_ready()+++def _pushd(path: Path) -> Iterator[None]:+ previous = Path.cwd()+ os.chdir(path)+ try:+ yield+ finally:+ os.chdir(previous)+++def _temporary_argv(argv: Sequence[str]) -> Iterator[None]:+ previous = list(sys.argv)+ sys.argv = list(argv)+ try:+ yield+ finally:+ sys.argv = previous+++def _print_banner_once(module) -> None:+ global _BANNER_PRINTED+ if _BANNER_PRINTED:+ return+ _BANNER_PRINTED = True+ banner = f"Powered by WanGP v{module.WanGP_version} - a DeepBeepMeep Production\n"+ console = sys.__stdout__ if sys.__stdout__ is not None else sys.stdout+ if console is not None:+ console.write(banner)+ console.flush()diff --git a/shared/kernels/quanto_int8_inject.py b/shared/kernels/quanto_int8_inject.pyindex e95deec..d71bae2 100644--- a/shared/kernels/quanto_int8_inject.py+++ b/shared/kernels/quanto_int8_inject.py@@ -39,7 +39,7 @@ _TIME_PROFILE_CPU_MS = 0.0_TIME_PROFILE_CALLS = 0_DEBUG_OVERRIDE: Optional[bool] = None-_PATCH_STATE = SimpleNamespace(enabled=False, orig_forward=None)+_PATCH_STATE = SimpleNamespace(enabled=False, orig_forward=None, orig_embedding_forward=None)_OPS_REGISTERED = False_OPS_NAMESPACE = "wan2gp_int8"_OPS_LIBS = []@@ -83,7 +83,7 @@ def _env_int(name: str, default: int) -> int:def _log(msg: str) -> None:- print(f"[WAN2GP][INT8][quanto] {msg}")+ print(f"[Quanto][INT8] {msg}")def _debug(msg: str) -> None:@@ -518,6 +518,68 @@ def _use_int8_kernel(input: torch.Tensor, other: torch.Tensor) -> bool:return input.is_cuda and input.dtype in (torch.bfloat16, torch.float16, torch.float32)+def _use_int8_embedding_kernel(module, input: torch.Tensor) -> bool:+ if _RUNTIME_DISABLED:+ return False+ if _TRITON_MODULE is None:+ return False+ if not torch.is_tensor(input):+ return False+ qweight = getattr(module, "qweight", None)+ if not _is_weight_qbytes(qweight):+ return False+ if qweight._data.dtype != torch.int8:+ return False+ if getattr(module, "max_norm", None) is not None:+ return False+ if bool(getattr(module, "scale_grad_by_freq", False)) or bool(getattr(module, "sparse", False)):+ return False+ if input.device != qweight._data.device:+ return False+ scale = getattr(qweight, "_scale", None)+ if not torch.is_tensor(scale) or scale.device != input.device:+ return False+ return True+++def _gather_embedding_scale(qweight, input: torch.Tensor) -> torch.Tensor:+ scale = qweight._scale+ if scale.ndim == 0 or scale.numel() == 1:+ return scale+ if scale.ndim == 1:+ if scale.shape[0] != qweight._data.shape[0]:+ raise RuntimeError("Quanto embedding scale length mismatch.")+ scale = scale.unsqueeze(-1)+ elif scale.ndim == 2:+ if scale.shape[0] != qweight._data.shape[0]:+ raise RuntimeError("Quanto embedding scale row count mismatch.")+ if scale.shape[1] != 1:+ raise RuntimeError("Quanto embedding fast path only supports per-row scales.")+ else:+ raise RuntimeError("Quanto embedding fast path only supports scalar or per-row scales.")+ return torch.nn.functional.embedding(input, scale)+++def _int8_embedding_forward(module, input: torch.Tensor) -> torch.Tensor:+ qweight = module.qweight+ gathered = torch.nn.functional.embedding(+ input,+ qweight._data,+ module.padding_idx,+ None,+ module.norm_type,+ False,+ False,+ )+ gathered = gathered.to(qweight._scale.dtype)+ scale = _gather_embedding_scale(qweight, input)+ if torch.is_tensor(scale):+ if scale.ndim == gathered.ndim - 1:+ scale = scale.unsqueeze(-1)+ return gathered * scale+ return gathered * scale++def _activation_rows(input_shape: torch.Size) -> int:rows = 1for dim in input_shape[:-1]:@@ -643,6 +705,11 @@ def enable_quanto_int8_kernel(triton_mod=None) -> bool:except Exception as exc:_debug(f"cannot import optimum.quanto qbytes ({exc})")return False+ try:+ from mmgp import offload as _mmgp_offload+ except Exception as exc:+ _debug(f"cannot import mmgp.offload ({exc})")+ return Falseif triton_mod is None:triton_mod, _ = _probe_triton_backend()@@ -655,6 +722,7 @@ def enable_quanto_int8_kernel(triton_mod=None) -> bool:_ensure_compile_safe_ops()orig_forward = _qbytes.WeightQBytesLinearFunction.forward+ orig_embedding_forward = _mmgp_offload.QEmbedding.forwarddef forward(ctx, input, other, bias=None):dense_hot_path = (@@ -706,9 +774,30 @@ def enable_quanto_int8_kernel(triton_mod=None) -> bool:f"Full Triton error details:\n{full_detail}") from exc+ def embedding_forward(self, input):+ if not _use_int8_embedding_kernel(self, input):+ return orig_embedding_forward(self, input)+ try:+ return _int8_embedding_forward(self, input)+ except Exception as exc:+ short_reason = _summarize_kernel_error(exc)+ if _allow_runtime_fallback():+ _disable_runtime(short_reason)+ _debug(f"Full embedding fast-path failure detail:\n{_format_exception_detail(exc)}")+ return orig_embedding_forward(self, input)+ full_detail = _format_exception_detail(exc)+ raise RuntimeError(+ "Injected Quanto int8 embedding fast path failed. "+ f"Set {_ENV_ALLOW_RUNTIME_FALLBACK}=1 to force fallback to non-injected Quanto path. "+ f"Reason: {short_reason}\n"+ f"Full error details:\n{full_detail}"+ ) from exc+_qbytes.WeightQBytesLinearFunction.forward = staticmethod(forward)+ _mmgp_offload.QEmbedding.forward = embedding_forward_PATCH_STATE.enabled = True_PATCH_STATE.orig_forward = orig_forward+ _PATCH_STATE.orig_embedding_forward = orig_embedding_forwardreturn True@@ -721,8 +810,12 @@ def disable_quanto_int8_kernel(notify_disabled = False) -> bool:from optimum.quanto.tensor.weights import qbytes as _qbytes_qbytes.WeightQBytesLinearFunction.forward = staticmethod(_PATCH_STATE.orig_forward)+ from mmgp import offload as _mmgp_offload+ if _PATCH_STATE.orig_embedding_forward is not None:+ _mmgp_offload.QEmbedding.forward = _PATCH_STATE.orig_embedding_forward_PATCH_STATE.enabled = False_PATCH_STATE.orig_forward = None+ _PATCH_STATE.orig_embedding_forward = None_FUSED_LAUNCH_CACHE = {}_FUSED_LAUNCH_CACHE_FIFO = []_SCALED_LAUNCH_CACHE = {}diff --git a/shared/kernels/quanto_int8_triton.py b/shared/kernels/quanto_int8_triton.pyindex 23d950f..4b72fa7 100644--- a/shared/kernels/quanto_int8_triton.py+++ b/shared/kernels/quanto_int8_triton.py@@ -36,6 +36,7 @@ _ENV_AUTOTUNE_MAX_REL_ERR = "WAN2GP_QUANTO_INT8_AUTOTUNE_MAX_REL_ERR"_ENV_AUTOTUNE_LOCK_FUSED_BLOCK_K = "WAN2GP_QUANTO_INT8_AUTOTUNE_LOCK_FUSED_BLOCK_K"_IS_AVAILABLE = None_CONFIG_LEN = 5+_AUTOTUNE_CACHE_VERSION = 2_AUTOTUNE_CACHE_LOADED = False_AUTOTUNE_CACHE_DIRTY = False_AUTOTUNE_CONFIG_CACHE: dict[str, tuple[int, int, int, int, int]] = {}@@ -94,6 +95,7 @@ _AUTOTUNE_SLOT_REPS = {"large_n_ge_2048": ((512, 4096, 4096), (3840, 3840, 4096)),"large_default": ((128, 4096, 1024), (192, 3072, 1536)),}+_RUNTIME_PROBE_MAX_N = 4096def _env_flag(name: str, default: str = "1") -> bool:@@ -303,7 +305,7 @@ def _save_autotune_cache() -> None:try:cache_path.parent.mkdir(parents=True, exist_ok=True)tmp_path = Path(f"{cache_path}.tmp")- payload = {"version": 1, "entries": {key: list(cfg) for key, cfg in _AUTOTUNE_CONFIG_CACHE.items()}}+ payload = {"version": _AUTOTUNE_CACHE_VERSION, "entries": {key: list(cfg) for key, cfg in _AUTOTUNE_CONFIG_CACHE.items()}}tmp_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")tmp_path.replace(cache_path)_AUTOTUNE_CACHE_DIRTY = False@@ -322,7 +324,7 @@ def _device_fingerprint(device_index: int) -> str:triton_ver = getattr(triton, "__version__", "0.0")return (f"{props.name}|cc={props.major}.{props.minor}|sm={props.multi_processor_count}|"- f"torch={torch.__version__}|triton={triton_ver}"+ f"torch={torch.__version__}|triton={triton_ver}|wan2gp_int8_cache_v={_AUTOTUNE_CACHE_VERSION}")@@ -358,6 +360,7 @@ def _set_cached_config(device_index: int, kernel_kind: str, slot_id: str, cfg: treturn_AUTOTUNE_CONFIG_CACHE[key] = cfg_AUTOTUNE_CACHE_DIRTY = True+ _save_autotune_cache()def _drop_cached_config(device_index: int, kernel_kind: str, slot_id: str, m: int, k: int, n: int) -> None:@@ -387,6 +390,32 @@ def _config_compatible_with_baseline(return True+def _runtime_probe_shape(+ kind: str,+ m: int,+ k: int,+ n: int,+ baseline: tuple[int, int, int, int, int],+) -> tuple[int, int, int, tuple[int, int, int, int, int]]:+ if n <= _RUNTIME_PROBE_MAX_N:+ return m, k, n, baseline+ probe_candidates = []+ for probe_n in (_RUNTIME_PROBE_MAX_N, 3072, 2048, 1536, 1024, 768, 512, 256, 128, 64):+ if probe_n >= n:+ continue+ if probe_n % 8 != 0:+ continue+ probe_candidates.append(probe_n)+ for probe_n in probe_candidates:+ probe_baseline = _select_static_triton_int8_config(m, k, probe_n)+ if not _config_compatible_with_baseline(kind, baseline, probe_baseline):+ continue+ if not _config_compatible_with_baseline(kind, probe_baseline, baseline):+ continue+ return m, k, probe_n, probe_baseline+ return m, k, n, baseline++def _candidate_configs(baseline: tuple[int, int, int, int, int],m: int,@@ -594,9 +623,10 @@ def _ensure_compile_compatible_config() -> tuple[tuple[int, int, int, int, int], Optional[Exception]]:device = torch.device("cuda", device_index)_ = rep_shapes- # Probing the current shape is enough to catch tile compile incompatibilities while- # keeping allocations low for large representative shapes.- probe_shapes = ((m, k, n),)+ probe_m, probe_k, probe_n, _ = _runtime_probe_shape(kind, m, k, n, baseline)+ # Probe a reduced runtime-compatible shape when possible so compile validation does+ # not allocate fake giant heads (for example lm_head with 248k vocab rows).+ probe_shapes = ((probe_m, probe_k, probe_n),)tensors_by_shape = {shape: _create_bench_tensors(kind, device, *shape) for shape in probe_shapes}candidates = _compile_recovery_candidates(kind, baseline, preferred, m, k, n)last_error: Optional[Exception] = None@@ -754,7 +784,8 @@ def _autotune_config(device = torch.device("cuda", device_index)cached = _get_cached_config(device_index, kind, slot_id, m, k, n)if cached is not None:- if _validate_config(kind, device, m, k, n, baseline, cached):+ probe_m, probe_k, probe_n, probe_baseline = _runtime_probe_shape(kind, m, k, n, baseline)+ if _validate_config(kind, device, probe_m, probe_k, probe_n, probe_baseline, cached):return cached_drop_cached_config(device_index, kind, slot_id, m, k, n)slot_key = (device_index, kind, slot_id)@@ -794,7 +825,8 @@ def _autotune_config(min_speedup = max(1.0, _env_float(_ENV_AUTOTUNE_MIN_SPEEDUP, 1.02))use_best = best_cfg != baseline and best_ms > 0.0 and (baseline_ms / best_ms) >= min_speeduppicked = best_cfg if use_best else baseline- if not _validate_config(kind, device, m, k, n, baseline, picked):+ probe_m, probe_k, probe_n, probe_baseline = _runtime_probe_shape(kind, m, k, n, baseline)+ if not _validate_config(kind, device, probe_m, probe_k, probe_n, probe_baseline, picked):picked = baseline_set_cached_config(device_index, kind, slot_id, picked)if use_best:@@ -830,13 +862,11 @@ def _select_triton_int8_config(cached = _AUTOTUNE_SESSION_CACHE.get(session_key)if cached is not None:return cached+ cached_cfg = _get_cached_config(device_index, kernel_kind, slot_id, m, k, n)+ if cached_cfg is not None and _config_compatible_with_baseline(kernel_kind, baseline, cached_cfg):+ _AUTOTUNE_SESSION_CACHE[session_key] = cached_cfg+ return cached_cfgif _is_stream_capturing():- _load_autotune_cache()- slot_key = _autotune_slot_cache_key(device_index, kernel_kind, slot_id)- cached_cfg = _AUTOTUNE_CONFIG_CACHE.get(slot_key)- if cached_cfg is not None and _config_compatible_with_baseline(kernel_kind, baseline, cached_cfg):- _AUTOTUNE_SESSION_CACHE[session_key] = cached_cfg- return cached_cfg# During graph capture we must avoid autotune/probing allocations. Do not populate# session cache with the baseline fallback, so a later non-capture call can autotune.return baselinediff --git a/shared/llm_engines/nanovllm/config.py b/shared/llm_engines/nanovllm/config.pyindex 55329ec..8545bff 100644--- a/shared/llm_engines/nanovllm/config.py+++ b/shared/llm_engines/nanovllm/config.py@@ -28,6 +28,7 @@ class Config:self.model_dir = self.modelassert self.kvcache_block_size % 256 == 0assert 1 <= self.tensor_parallel_size <= 8- self.hf_config = AutoConfig.from_pretrained(self.model_dir)+ if self.hf_config is None:+ self.hf_config = AutoConfig.from_pretrained(self.model_dir)self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings)assert self.max_num_batched_tokens >= self.max_model_lendiff --git a/shared/llm_engines/nanovllm/engine/llm_engine.py b/shared/llm_engines/nanovllm/engine/llm_engine.pyindex 8c42fdf..71f99f0 100644--- a/shared/llm_engines/nanovllm/engine/llm_engine.py+++ b/shared/llm_engines/nanovllm/engine/llm_engine.py@@ -125,7 +125,18 @@ class LLMEngine:returnrunner.clear_graph_cache()- def add_request(self, prompt: str | list[int], sampling_params: SamplingParams, unconditional_prompt: str | list[int] | None = None):+ def add_request(+ self,+ prompt: str | list[int],+ sampling_params: SamplingParams,+ unconditional_prompt: str | list[int] | None = None,+ prompt_embeds=None,+ prompt_position_ids=None,+ position_offset: int = 0,+ unconditional_prompt_embeds=None,+ unconditional_prompt_position_ids=None,+ unconditional_position_offset: int = 0,+ ):if isinstance(prompt, str):prompt = self.tokenizer.encode(prompt)# For CFG: if cfg_scale > 1.0, create both conditional and unconditional sequences@@ -142,15 +153,36 @@ class LLMEngine:if isinstance(unconditional_prompt, str):unconditional_prompt = self.tokenizer.encode(unconditional_prompt)# Create unconditional sequence first (so we can reference it from conditional)- uncond_seq = Sequence(unconditional_prompt, sampling_params, is_unconditional=True)+ uncond_seq = Sequence(+ unconditional_prompt,+ sampling_params,+ is_unconditional=True,+ prompt_embeds=unconditional_prompt_embeds,+ prompt_position_ids=unconditional_prompt_position_ids,+ position_offset=unconditional_position_offset,+ )# Create conditional sequence with reference to unconditional- cond_seq = Sequence(prompt, sampling_params, is_unconditional=False, conditional_seq=uncond_seq)+ cond_seq = Sequence(+ prompt,+ sampling_params,+ is_unconditional=False,+ conditional_seq=uncond_seq,+ prompt_embeds=prompt_embeds,+ prompt_position_ids=prompt_position_ids,+ position_offset=position_offset,+ )uncond_seq.paired_seq = cond_seq # Link them bidirectionally# Add both sequences to schedulerself.scheduler.add(cond_seq)self.scheduler.add(uncond_seq)else:- seq = Sequence(prompt, sampling_params)+ seq = Sequence(+ prompt,+ sampling_params,+ prompt_embeds=prompt_embeds,+ prompt_position_ids=prompt_position_ids,+ position_offset=position_offset,+ )self.scheduler.add(seq)def step(self):@@ -259,3 +291,80 @@ class LLMEngine:outputs = [outputs[seq_id] for seq_id in sorted(outputs.keys())]outputs = [{"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} for token_ids in outputs]return outputs++ def generate_embedded(+ self,+ prompts: list[list[int]],+ prompt_embeds: list,+ prompt_position_ids: list | None,+ sampling_params: SamplingParams | list[SamplingParams],+ position_offsets: list[int] | None = None,+ use_tqdm: bool = True,+ ):+ if self.scheduler is None:+ raise RuntimeError("LLM engine is closed.")+ self.model_runner.ensure_runtime_ready()+ if (+ self.config.num_kvcache_blocks > 0+ and len(self.scheduler.block_manager.blocks) != self.config.num_kvcache_blocks+ ):+ self.scheduler.block_manager = BlockManager(+ self.config.num_kvcache_blocks,+ self.config.kvcache_block_size,+ )+ if not self.is_finished():+ self.reset()+ if not isinstance(sampling_params, list):+ sampling_params = [sampling_params] * len(prompts)+ if prompt_position_ids is None:+ prompt_position_ids = [None] * len(prompts)+ if position_offsets is None:+ position_offsets = [0] * len(prompts)+ if use_tqdm:+ pbar = tqdm(total=len(prompts), desc="Generating", dynamic_ncols=True)+ seed_to_apply = None+ for sp in sampling_params:+ seed_val = getattr(sp, "seed", None)+ if seed_val is not None:+ try:+ seed_to_apply = int(seed_val)+ break+ except Exception:+ seed_to_apply = None+ self.model_runner.call("set_sampling_seed", seed_to_apply)+ for prompt, embeds, pos_ids, pos_offset, sp in zip(prompts, prompt_embeds, prompt_position_ids, position_offsets, sampling_params):+ self.add_request(+ prompt,+ sp,+ prompt_embeds=embeds,+ prompt_position_ids=pos_ids,+ position_offset=pos_offset,+ )+ outputs = {}+ prefill_throughput = decode_throughput = 0.0+ try:+ while not self.is_finished():+ t = perf_counter()+ output, num_tokens = self.step()+ if use_tqdm:+ if num_tokens > 0:+ prefill_throughput = num_tokens / (perf_counter() - t)+ else:+ decode_throughput = -num_tokens / (perf_counter() - t)+ pbar.set_postfix({+ "Prefill": f"{int(prefill_throughput)}tok/s",+ "Decode": f"{int(decode_throughput)}tok/s",+ })+ for seq_id, token_ids in output:+ outputs[seq_id] = token_ids+ if use_tqdm:+ pbar.update(1)+ except Exception:+ self.reset()+ raise+ finally:+ if use_tqdm:+ pbar.close()+ outputs = [outputs[seq_id] for seq_id in sorted(outputs.keys())]+ outputs = [{"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} for token_ids in outputs]+ return outputsdiff --git a/shared/llm_engines/nanovllm/engine/model_runner.py b/shared/llm_engines/nanovllm/engine/model_runner.pyindex aa1e12a..405daa3 100644--- a/shared/llm_engines/nanovllm/engine/model_runner.py+++ b/shared/llm_engines/nanovllm/engine/model_runner.py@@ -128,10 +128,20 @@ class ModelRunner:raise RuntimeError("LLM model object is not available.")self._tie_word_embeddings_if_needed()self.allocate_kv_cache()+ self._prepare_model_sequence_state()if not self.enforce_eager:self.capture_cudagraph()self._runtime_ready = True+ def _prepare_model_sequence_state(self):+ if self.model is None:+ return+ model_device = self._get_model_device()+ for module in self.model.modules():+ prepare = getattr(module, "prepare_sequence_state", None)+ if callable(prepare):+ prepare(self.config.max_num_seqs, model_device, self.dtype)+def _get_tied_embeddings(self):if self.model is None:return None@@ -167,6 +177,18 @@ class ModelRunner:for module in self.model.modules():if hasattr(module, "k_cache") and hasattr(module, "v_cache"):module.k_cache = module.v_cache = torch.tensor([])+ release_sequence_state = getattr(module, "release_sequence_state", None)+ if callable(release_sequence_state):+ release_sequence_state()+ continue+ reset_sequence_state = getattr(module, "reset_sequence_state", None)+ if callable(reset_sequence_state):+ reset_sequence_state()+ else:+ if hasattr(module, "conv_state"):+ module.conv_state = None+ if hasattr(module, "recurrent_state"):+ module.recurrent_state = Noneexcept Exception:passif hasattr(self, "kv_cache"):@@ -181,16 +203,23 @@ class ModelRunner:except Exception:passtry:- self.graphs = {}- self.graph_vars = {}- self.graph_bs = []- self.graph_pool = None+ for attr_name in ("graphs", "graph_vars", "graph_bs", "graph_pool"):+ if hasattr(self, attr_name):+ delattr(self, attr_name)+ except Exception:+ pass+ try:+ torch.cuda.synchronize()except Exception:passtry:torch.cuda.empty_cache()except Exception:pass+ try:+ torch.cuda.ipc_collect()+ except Exception:+ passself._logits_bias_cache.clear()self._sampling_generator = Noneself._runtime_ready = False@@ -354,6 +383,10 @@ class ModelRunner:if not self.enforce_eager:if hasattr(self, "graphs"):del self.graphs+ if hasattr(self, "graph_vars"):+ del self.graph_vars+ if hasattr(self, "graph_bs"):+ del self.graph_bsif hasattr(self, "graph_pool"):del self.graph_pooltry:@@ -364,6 +397,10 @@ class ModelRunner:torch.cuda.empty_cache()except Exception:pass+ try:+ torch.cuda.ipc_collect()+ except Exception:+ passif dist.is_initialized():dist.destroy_process_group()@@ -456,8 +493,13 @@ class ModelRunner:return block_tablesdef prepare_prefill(self, seqs: list[Sequence]):+ use_prompt_embeds = any(getattr(seq, "prompt_embeds", None) is not None for seq in seqs)+ if use_prompt_embeds and not all(getattr(seq, "prompt_embeds", None) is not None for seq in seqs):+ raise RuntimeError("Mixed embedded/non-embedded prefill batches are not supported.")input_ids = []positions = []+ prompt_embeds = []+ prompt_position_ids = []cu_seqlens_q = [0]cu_seqlens_k = [0]max_seqlen_q = 0@@ -467,9 +509,27 @@ class ModelRunner:for seq in seqs:seqlen = len(seq)input_ids.extend(seq[seq.num_cached_tokens:])- positions.extend(list(range(seq.num_cached_tokens, seqlen)))seqlen_q = seqlen - seq.num_cached_tokensseqlen_k = seqlen+ if use_prompt_embeds:+ seq_prompt_embeds = getattr(seq, "prompt_embeds", None)+ seq_position_ids = getattr(seq, "prompt_position_ids", None)+ if seq_prompt_embeds is None or seq_position_ids is None:+ raise RuntimeError("Embedded prefill requires both prompt_embeds and prompt_position_ids.")+ seq_prompt_embeds = seq_prompt_embeds[seq.num_cached_tokens:seqlen]+ if seq_prompt_embeds.ndim != 2 or seq_prompt_embeds.shape[0] != seqlen_q:+ raise RuntimeError("Embedded prefill prompt_embeds shape does not match uncached prompt length.")+ if seq_position_ids.ndim == 3:+ seq_position_ids = seq_position_ids[:, 0]+ if seq_position_ids.ndim != 2 or seq_position_ids.shape[0] != 3:+ raise RuntimeError("Embedded prefill prompt_position_ids must have shape [3, seq_len].")+ seq_position_ids = seq_position_ids[:, seq.num_cached_tokens:seqlen]+ if seq_position_ids.shape[1] != seqlen_q:+ raise RuntimeError("Embedded prefill prompt_position_ids shape does not match uncached prompt length.")+ prompt_embeds.append(seq_prompt_embeds)+ prompt_position_ids.append(seq_position_ids)+ else:+ positions.extend(list(range(seq.num_cached_tokens, seqlen)))cu_seqlens_q.append(cu_seqlens_q[-1] + seqlen_q)cu_seqlens_k.append(cu_seqlens_k[-1] + seqlen_k)max_seqlen_q = max(seqlen_q, max_seqlen_q)@@ -486,13 +546,19 @@ class ModelRunner:slot_mapping.extend(list(range(start, end)))if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cacheblock_tables = self.prepare_block_tables(seqs)- input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True)- positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True)+ if use_prompt_embeds:+ input_ids = None+ positions = torch.cat(prompt_position_ids, dim=1).unsqueeze(1).contiguous()+ inputs_embeds = torch.cat(prompt_embeds, dim=0).unsqueeze(0).contiguous()+ else:+ input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True)+ positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True)+ inputs_embeds = Nonecu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True)cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True)slot_mapping = torch.tensor(slot_mapping, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True)set_context(True, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, slot_mapping, None, block_tables)- return input_ids, positions+ return input_ids, positions, inputs_embedsdef prepare_decode(self, seqs: list[Sequence]):"""Optimized decode preparation using pre-allocated buffers."""@@ -501,7 +567,7 @@ class ModelRunner:# Use pre-allocated CPU buffersfor i, seq in enumerate(seqs):self._cpu_input_ids[i] = seq.last_token- self._cpu_positions[i] = len(seq) - 1+ self._cpu_positions[i] = len(seq) - 1 + int(getattr(seq, "position_offset", 0) or 0)self._cpu_context_lens[i] = len(seq)self._cpu_slot_mapping[i] = seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1@@ -550,11 +616,15 @@ class ModelRunner:return temperatures, cfg_scales, top_ks, top_ps, repetition_penalties@torch.inference_mode()- def run_model(self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool):- if is_prefill or self.enforce_eager or input_ids.size(0) > 512:- return self.model.compute_logits(self.model(input_ids, positions))+ def run_model(self, input_ids: torch.Tensor | None, positions: torch.Tensor, is_prefill: bool, inputs_embeds: torch.Tensor | None = None):+ decode_batch_size = input_ids.size(0) if input_ids is not None else int(inputs_embeds.shape[0])+ model_kwargs = {"input_ids": input_ids, "positions": positions}+ if inputs_embeds is not None:+ model_kwargs["inputs_embeds"] = inputs_embeds+ if is_prefill or self.enforce_eager or decode_batch_size > 512:+ return self.model.compute_logits(self.model(**model_kwargs))else:- bs = input_ids.size(0)+ bs = decode_batch_sizecontext = get_context()# Check if block_tables size exceeds pre-allocated buffer size@@ -567,7 +637,7 @@ class ModelRunner:"cudagraph_fallback_block_table_cols",f"requested={context.block_tables.size(1)} max={max_num_blocks}",)- return self.model.compute_logits(self.model(input_ids, positions))+ return self.model.compute_logits(self.model(**model_kwargs))# Fix: Also check if block_tables row count matches batch size# Dimension mismatch can cause CUDA illegal memory access during graph replay@@ -577,7 +647,7 @@ class ModelRunner:"cudagraph_fallback_block_table_rows",f"rows={context.block_tables.size(0)} bs={bs}",)- return self.model.compute_logits(self.model(input_ids, positions))+ return self.model.compute_logits(self.model(**model_kwargs))# Fix: Verify slot_mapping and context_lens dimensions match batch sizeif context.slot_mapping.size(0) != bs or context.context_lens.size(0) != bs:@@ -586,7 +656,7 @@ class ModelRunner:"cudagraph_fallback_context_shape",f"slot={context.slot_mapping.size(0)} ctx={context.context_lens.size(0)} bs={bs}",)- return self.model.compute_logits(self.model(input_ids, positions))+ return self.model.compute_logits(self.model(**model_kwargs))graph = self.graphs[next(x for x in self.graph_bs if x >= bs)]graph_vars = self.graph_vars@@ -616,7 +686,11 @@ class ModelRunner:# uncond_seqs = seqs[num_cond:]# Prepare inputs for both conditional and unconditional (they're already in the batch)- input_ids, positions = (self.prepare_prefill(seqs) if is_prefill else self.prepare_decode(seqs))+ if is_prefill:+ input_ids, positions, inputs_embeds = self.prepare_prefill(seqs)+ else:+ input_ids, positions = self.prepare_decode(seqs)+ inputs_embeds = Nonesample_params = self.prepare_sample(seqs, is_cfg_batch=True) if self.rank == 0 else Noneif sample_params is not None:temperatures, cfg_scales, top_ks, top_ps, repetition_penalties = sample_params@@ -624,7 +698,10 @@ class ModelRunner:temperatures = cfg_scales = top_ks = top_ps = repetition_penalties = None# Run model forward (processes entire batch: cond + uncond)- logits_all = self.run_model(input_ids, positions, is_prefill)+ logits_all = self.run_model(input_ids, positions, is_prefill, inputs_embeds=inputs_embeds)+ if is_prefill:+ for seq in seqs:+ seq.clear_prompt_data()reset_context()if self.rank == 0:@@ -698,14 +775,20 @@ class ModelRunner:return Noneelse:# Normal batch (non-CFG)- input_ids, positions = (self.prepare_prefill(seqs) if is_prefill- else self.prepare_decode(seqs))+ if is_prefill:+ input_ids, positions, inputs_embeds = self.prepare_prefill(seqs)+ else:+ input_ids, positions = self.prepare_decode(seqs)+ inputs_embeds = Nonesample_params = self.prepare_sample(seqs, is_cfg_batch=False) if self.rank == 0 else Noneif sample_params is not None:temperatures, cfg_scales, top_ks, top_ps, repetition_penalties = sample_paramselse:temperatures = cfg_scales = top_ks = top_ps = repetition_penalties = None- logits = self.run_model(input_ids, positions, is_prefill)+ logits = self.run_model(input_ids, positions, is_prefill, inputs_embeds=inputs_embeds)+ if is_prefill:+ for seq in seqs:+ seq.clear_prompt_data()reset_context()if self.rank == 0:@@ -797,7 +880,12 @@ class ModelRunner:context_lens = torch.zeros(max_bs, dtype=torch.int32, device=model_device)block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32, device=model_device)outputs = torch.zeros(max_bs, hf_config.hidden_size, device=model_device)- self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16))+ base_graph_bs = [1, 2, 4, 8]+ self.graph_bs = [bs for bs in base_graph_bs if bs <= max_bs]+ if max_bs > 8:+ self.graph_bs.extend(range(16, max_bs + 1, 16))+ if not self.graph_bs:+ self.graph_bs = [max_bs]self.graphs = {}self.graph_pool = Nonediff --git a/shared/llm_engines/nanovllm/engine/sequence.py b/shared/llm_engines/nanovllm/engine/sequence.pyindex 86869d5..756f8b0 100644--- a/shared/llm_engines/nanovllm/engine/sequence.py+++ b/shared/llm_engines/nanovllm/engine/sequence.py@@ -16,7 +16,16 @@ class Sequence:block_size = 256counter = count()- def __init__(self, token_ids: list[int], sampling_params = SamplingParams(), is_unconditional: bool = False, conditional_seq = None):+ def __init__(+ self,+ token_ids: list[int],+ sampling_params=SamplingParams(),+ is_unconditional: bool = False,+ conditional_seq=None,+ prompt_embeds=None,+ prompt_position_ids=None,+ position_offset: int = 0,+ ):self.seq_id = next(Sequence.counter)self.status = SequenceStatus.WAITINGself.token_ids = copy(token_ids)@@ -41,6 +50,9 @@ class Sequence:self.logits_processor: Optional[Any] = sampling_params.logits_processorself.logits_processor_update_state: Optional[Callable[[int], None]] = sampling_params.logits_processor_update_stateself.logits_bias: Optional[Any] = sampling_params.logits_bias+ self.prompt_embeds = prompt_embeds+ self.prompt_position_ids = prompt_position_ids+ self.position_offset = int(position_offset or 0)def __len__(self):return self.num_tokens@@ -85,6 +97,10 @@ class Sequence:self.last_token = token_idself.num_tokens += 1+ def clear_prompt_data(self):+ self.prompt_embeds = None+ self.prompt_position_ids = None+def __getstate__(self):return (self.num_tokens, self.num_prompt_tokens, self.num_cached_tokens, self.block_table,self.token_ids if self.num_completion_tokens == 0 else self.last_token)diff --git a/shared/llm_engines/nanovllm/layers/layernorm.py b/shared/llm_engines/nanovllm/layers/layernorm.pyindex 71bf419..676faef 100644--- a/shared/llm_engines/nanovllm/layers/layernorm.py+++ b/shared/llm_engines/nanovllm/layers/layernorm.py@@ -1,6 +1,61 @@+from __future__ import annotations+import torchfrom torch import nn+try:+ import triton+ import triton.language as tl+except Exception: # pragma: no cover+ triton = None+ tl = None+++if triton is not None:+ @triton.jit+ def _rmsnorm_kernel(+ x_ptr,+ residual_ptr,+ weight_ptr,+ y_ptr,+ residual_out_ptr,+ x_row_stride,+ residual_row_stride,+ y_row_stride,+ residual_out_row_stride,+ n_cols,+ eps,+ HAS_RESIDUAL: tl.constexpr,+ STORE_RESIDUAL: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ ):+ row = tl.program_id(0)+ offs = tl.arange(0, BLOCK_SIZE)+ mask = offs < n_cols++ x = tl.load(x_ptr + row * x_row_stride + offs, mask=mask, other=0.0).to(tl.float32)+ if HAS_RESIDUAL:+ residual = tl.load(+ residual_ptr + row * residual_row_stride + offs,+ mask=mask,+ other=0.0,+ ).to(tl.float32)+ x = x + residual+ if STORE_RESIDUAL:+ tl.store(residual_out_ptr + row * residual_out_row_stride + offs, x, mask=mask)++ var = tl.sum(x * x, axis=0) / n_cols+ inv_rms = tl.rsqrt(var + eps)+ weight = tl.load(weight_ptr + offs, mask=mask, other=1.0).to(tl.float32)+ y = x * inv_rms * weight+ tl.store(y_ptr + row * y_row_stride + offs, y, mask=mask)+++def _next_power_of_2(n: int) -> int:+ if n <= 1:+ return 1+ return 1 << (int(n) - 1).bit_length()+class RMSNorm(nn.Module):@@ -13,31 +68,87 @@ class RMSNorm(nn.Module):self.eps = epsself.weight = nn.Parameter(torch.ones(hidden_size))- @torch.compile- def rms_forward(+ def _can_use_triton(self, x: torch.Tensor, residual: torch.Tensor | None = None) -> bool:+ if triton is None or tl is None:+ return False+ if not x.is_cuda or self.weight.device.type != "cuda":+ return False+ if x.stride(-1) != 1 or self.weight.stride(-1) != 1:+ return False+ if x.dtype not in (torch.float16, torch.bfloat16, torch.float32):+ return False+ if residual is not None:+ if residual.device != x.device or residual.dtype != x.dtype or residual.stride(-1) != 1:+ return False+ return int(x.shape[-1]) <= 8192++ def _fallback_rms_forward(self, x: torch.Tensor) -> torch.Tensor:+ orig_dtype = x.dtype+ x_float = x.float()+ var = x_float.pow(2).mean(dim=-1, keepdim=True)+ return (x_float * torch.rsqrt(var + self.eps)).to(orig_dtype) * self.weight++ def _fallback_add_rms_forward(self,x: torch.Tensor,- ) -> torch.Tensor:+ residual: torch.Tensor,+ ) -> tuple[torch.Tensor, torch.Tensor]:orig_dtype = x.dtype- x = x.float()- var = x.pow(2).mean(dim=-1, keepdim=True)- x.mul_(torch.rsqrt(var + self.eps))- x = x.to(orig_dtype).mul_(self.weight)- return x-- @torch.compile- def add_rms_forward(+ summed = x.float() + residual.float()+ residual_out = summed.to(orig_dtype)+ var = summed.pow(2).mean(dim=-1, keepdim=True)+ normed = (summed * torch.rsqrt(var + self.eps)).to(orig_dtype) * self.weight+ return normed, residual_out++ def _triton_rms_forward(self, x: torch.Tensor) -> torch.Tensor:+ x_2d = x.reshape(-1, x.shape[-1])+ y_2d = torch.empty_like(x_2d)+ block_size = _next_power_of_2(int(x_2d.shape[1]))+ _rmsnorm_kernel[(x_2d.shape[0],)](+ x_2d,+ x_2d,+ self.weight,+ y_2d,+ y_2d,+ x_2d.stride(0),+ x_2d.stride(0),+ y_2d.stride(0),+ y_2d.stride(0),+ x_2d.shape[1],+ self.eps,+ HAS_RESIDUAL=False,+ STORE_RESIDUAL=False,+ BLOCK_SIZE=block_size,+ )+ return y_2d.view_as(x)++ def _triton_add_rms_forward(self,x: torch.Tensor,residual: torch.Tensor,) -> tuple[torch.Tensor, torch.Tensor]:- orig_dtype = x.dtype- x = x.float().add_(residual.float())- residual = x.to(orig_dtype)- var = x.pow(2).mean(dim=-1, keepdim=True)- x.mul_(torch.rsqrt(var + self.eps))- x = x.to(orig_dtype).mul_(self.weight)- return x, residual+ x_2d = x.reshape(-1, x.shape[-1])+ residual_2d = residual.reshape(-1, residual.shape[-1])+ y_2d = torch.empty_like(x_2d)+ residual_out_2d = torch.empty_like(x_2d)+ block_size = _next_power_of_2(int(x_2d.shape[1]))+ _rmsnorm_kernel[(x_2d.shape[0],)](+ x_2d,+ residual_2d,+ self.weight,+ y_2d,+ residual_out_2d,+ x_2d.stride(0),+ residual_2d.stride(0),+ y_2d.stride(0),+ residual_out_2d.stride(0),+ x_2d.shape[1],+ self.eps,+ HAS_RESIDUAL=True,+ STORE_RESIDUAL=True,+ BLOCK_SIZE=block_size,+ )+ return y_2d.view_as(x), residual_out_2d.view_as(x)def forward(self,@@ -45,6 +156,10 @@ class RMSNorm(nn.Module):residual: torch.Tensor | None = None,) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:if residual is None:- return self.rms_forward(x)- else:- return self.add_rms_forward(x, residual)+ if self._can_use_triton(x):+ return self._triton_rms_forward(x)+ return self._fallback_rms_forward(x)++ if self._can_use_triton(x, residual):+ return self._triton_add_rms_forward(x, residual)+ return self._fallback_add_rms_forward(x, residual)diff --git a/shared/llm_engines/nanovllm/layers/sampler.py b/shared/llm_engines/nanovllm/layers/sampler.pyindex 8753882..4aecf5d 100644--- a/shared/llm_engines/nanovllm/layers/sampler.py+++ b/shared/llm_engines/nanovllm/layers/sampler.py@@ -16,11 +16,21 @@ def apply_top_k_top_p(The logits tensor is updated in-place."""+ squeezed = False+ if logits.ndim == 1:+ logits = logits.unsqueeze(0)+ squeezed = True+ if k is not None:+ k = k.reshape(-1)+ if p is not None:+ p = p.reshape(-1)+if p is None:if k is None:- return logits+ return logits.squeeze(0) if squeezed else logits# Avoid sorting vocab for top-k only case- return apply_top_k_only(logits, k)+ logits = apply_top_k_only(logits, k)+ return logits.squeeze(0) if squeezed else logits# Need to sort for top-plogits_sort, logits_idx = logits.sort(dim=-1, descending=False)@@ -46,7 +56,7 @@ def apply_top_k_top_p(# Re-sort back to original positionslogits.scatter_(dim=-1, index=logits_idx, src=logits_sort)- return logits+ return logits.squeeze(0) if squeezed else logitsdef apply_top_k_only(@@ -58,6 +68,12 @@ def apply_top_k_only(This is much faster than sorting for top-k only cases.The logits tensor is updated in-place."""+ squeezed = False+ if logits.ndim == 1:+ logits = logits.unsqueeze(0)+ squeezed = True+ k = k.reshape(-1)+vocab_size = logits.shape[1]# Handle cases where k >= vocab_size (no filtering needed)no_top_k_mask = (k <= 0) | (k >= vocab_size)@@ -81,7 +97,7 @@ def apply_top_k_only(# Mask all values below the thresholdlogits.masked_fill_(logits < top_k_thresh, float('-inf'))- return logits+ return logits.squeeze(0) if squeezed else logitsclass Sampler(nn.Module):@@ -120,5 +136,5 @@ class Sampler(nn.Module):logits[invalid_rows, 0] = 0.0probs = torch.softmax(logits, dim=-1)noise = torch.empty_like(probs).exponential_(1, generator=generator).clamp_min_(1e-10)- sample_tokens = probs.div_(noise).argmax(dim=-1)+ sample_tokens = probs.div_(noise).argmax(dim=-1).reshape(-1)return sample_tokensdiff --git a/shared/llm_engines/nanovllm/models/__init__.py b/shared/llm_engines/nanovllm/models/__init__.pyindex 67d5754..802c24a 100644--- a/shared/llm_engines/nanovllm/models/__init__.py+++ b/shared/llm_engines/nanovllm/models/__init__.py@@ -1,3 +1,9 @@from .qwen3 import Qwen3ForCausalLM+from .qwen3_5 import Qwen3_5DynamicCache, Qwen3_5ForCausalLM, clear_qwen35_runtime_caches-__all__ = ["Qwen3ForCausalLM"]+__all__ = [+ "Qwen3ForCausalLM",+ "Qwen3_5DynamicCache",+ "Qwen3_5ForCausalLM",+ "clear_qwen35_runtime_caches",+]diff --git a/shared/llm_engines/nanovllm/models/qwen3_5.py b/shared/llm_engines/nanovllm/models/qwen3_5.pynew file mode 100644index 0000000..397e455--- /dev/null+++ b/shared/llm_engines/nanovllm/models/qwen3_5.py@@ -0,0 +1,940 @@+from __future__ import annotations++import logging+import torch+import torch.distributed as dist+import torch.nn.functional as F+from torch import nn++logging.getLogger("fla.utils").setLevel(logging.ERROR)++try:+ from flash_attn import flash_attn_varlen_func+except Exception: # pragma: no cover+ flash_attn_varlen_func = None++try:+ # from causal_conv1d import causal_conv1d_fn, causal_conv1d_update+ causal_conv1d_fn = None+ causal_conv1d_update = None+except Exception: # pragma: no cover+ causal_conv1d_fn = None+ causal_conv1d_update = None++try:+ from fla.ops.gated_delta_rule import chunk_gated_delta_rule as fast_chunk_gated_delta_rule+ from fla.ops.gated_delta_rule import fused_recurrent_gated_delta_rule as fast_recurrent_gated_delta_rule+except Exception: # pragma: no cover+ fast_chunk_gated_delta_rule = None+ fast_recurrent_gated_delta_rule = None++try:+ from fla.modules import FusedRMSNormGated, ShortConvolution+except Exception: # pragma: no cover+ FusedRMSNormGated = None+ ShortConvolution = None++_DEFAULT_FLASH_ATTN_VARLEN_FUNC = flash_attn_varlen_func+_DEFAULT_FAST_CHUNK_GATED_DELTA_RULE = fast_chunk_gated_delta_rule+_DEFAULT_FAST_RECURRENT_GATED_DELTA_RULE = fast_recurrent_gated_delta_rule+_DEFAULT_FUSED_RMSNORM_GATED = FusedRMSNormGated+_DEFAULT_SHORT_CONVOLUTION = ShortConvolution+++def configure_qwen35_safe_legacy_kernels(enabled: bool) -> None:+ global flash_attn_varlen_func, fast_chunk_gated_delta_rule, fast_recurrent_gated_delta_rule+ global FusedRMSNormGated, ShortConvolution+ if enabled:+ flash_attn_varlen_func = None+ fast_chunk_gated_delta_rule = None+ fast_recurrent_gated_delta_rule = None+ FusedRMSNormGated = None+ ShortConvolution = None+ return+ flash_attn_varlen_func = _DEFAULT_FLASH_ATTN_VARLEN_FUNC+ fast_chunk_gated_delta_rule = _DEFAULT_FAST_CHUNK_GATED_DELTA_RULE+ fast_recurrent_gated_delta_rule = _DEFAULT_FAST_RECURRENT_GATED_DELTA_RULE+ FusedRMSNormGated = _DEFAULT_FUSED_RMSNORM_GATED+ ShortConvolution = _DEFAULT_SHORT_CONVOLUTION++from ..layers.activation import SiluAndMul+from ..layers.attention import Attention+from ..layers.layernorm import RMSNorm+from ..layers.linear import ColumnParallelLinear, RowParallelLinear+from ..utils.context import get_context++_CU_SEQLENS_CACHE: dict[tuple[tuple[int, ...], str], torch.Tensor] = {}+_MROPE_FREQ_CACHE: dict[tuple[str, int, int, float], torch.Tensor] = {}+++def _get_tp_size() -> int:+ if dist.is_available() and dist.is_initialized():+ return dist.get_world_size()+ return 1+++def _build_cu_seqlens(lengths: list[int], device: torch.device) -> torch.Tensor:+ normalized_lengths = tuple(int(length) for length in lengths)+ cache_key = (normalized_lengths, str(device))+ cached = _CU_SEQLENS_CACHE.get(cache_key)+ if cached is not None:+ return cached++ total = 0+ out = [0]+ for length in normalized_lengths:+ total += length+ out.append(total)+ cached = torch.tensor(out, dtype=torch.int32, device=device)+ _CU_SEQLENS_CACHE[cache_key] = cached+ return cached+++def _build_mrope_freq_cache(+ device: torch.device,+ max_position: int,+ rotary_dim: int,+ rope_theta: float,+) -> torch.Tensor:+ cache_key = (str(device), int(max_position), int(rotary_dim), float(rope_theta))+ cached = _MROPE_FREQ_CACHE.get(cache_key)+ if cached is not None:+ return cached++ inv_freq = 1.0 / (+ rope_theta ** (torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device) / rotary_dim)+ )+ positions = torch.arange(max_position, dtype=torch.float32, device=device)+ cached = torch.einsum("i,j->ij", positions, inv_freq)+ _MROPE_FREQ_CACHE[cache_key] = cached+ return cached+++def _interleave_last_dim_halves(tensor: torch.Tensor) -> torch.Tensor:+ if tensor.shape[-1] % 2 != 0:+ return tensor+ first_half, second_half = tensor.chunk(2, dim=-1)+ return torch.stack((first_half, second_half), dim=-1).flatten(-2, -1)+++def clear_qwen35_runtime_caches(device: torch.device | None = None) -> None:+ if device is None:+ _CU_SEQLENS_CACHE.clear()+ _MROPE_FREQ_CACHE.clear()+ return++ device_key = str(device)+ for cache_key in [key for key in _CU_SEQLENS_CACHE if key[1] == device_key]:+ _CU_SEQLENS_CACHE.pop(cache_key, None)+ for cache_key in [key for key in _MROPE_FREQ_CACHE if key[0] == device_key]:+ _MROPE_FREQ_CACHE.pop(cache_key, None)+++def rotate_half(x: torch.Tensor) -> torch.Tensor:+ x1 = x[..., : x.shape[-1] // 2]+ x2 = x[..., x.shape[-1] // 2 :]+ return torch.cat((-x2, x1), dim=-1)+++def apply_rotary_pos_emb(+ q: torch.Tensor,+ k: torch.Tensor,+ cos: torch.Tensor,+ sin: torch.Tensor,+) -> tuple[torch.Tensor, torch.Tensor]:+ rotary_dim = cos.shape[-1]+ q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]+ k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]+ cos = cos.unsqueeze(2)+ sin = sin.unsqueeze(2)+ q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin)+ k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin)+ return torch.cat((q_embed, q_pass), dim=-1), torch.cat((k_embed, k_pass), dim=-1)+++def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6) -> torch.Tensor:+ return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)+++def torch_causal_conv1d_update(+ hidden_states: torch.Tensor,+ conv_state: torch.Tensor,+ weight: torch.Tensor,+ bias: torch.Tensor | None = None,+):+ _, hidden_size, seq_len = hidden_states.shape+ state_len = conv_state.shape[-1]+ hidden_states_new = torch.cat([conv_state, hidden_states], dim=-1).to(weight.dtype)+ conv_state.copy_(hidden_states_new[:, :, -state_len:])+ out = F.conv1d(hidden_states_new, weight.unsqueeze(1), bias=bias, padding=0, groups=hidden_size)+ out = F.silu(out[:, :, -seq_len:])+ return out.to(hidden_states.dtype)+++def torch_chunk_gated_delta_rule(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ chunk_size: int = 64,+ initial_state: torch.Tensor | None = None,+ output_final_state: bool = False,+) -> tuple[torch.Tensor, torch.Tensor | None]:+ initial_dtype = query.dtype+ query = l2norm(query, dim=-1)+ key = l2norm(key, dim=-1)+ query, key, value, beta, g = [+ x.transpose(1, 2).contiguous().to(torch.float32) for x in (query, key, value, beta, g)+ ]++ batch_size, num_heads, sequence_length, k_head_dim = key.shape+ v_head_dim = value.shape[-1]+ pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size+ query = F.pad(query, (0, 0, 0, pad_size))+ key = F.pad(key, (0, 0, 0, pad_size))+ value = F.pad(value, (0, 0, 0, pad_size))+ beta = F.pad(beta, (0, pad_size))+ g = F.pad(g, (0, pad_size))+ total_sequence_length = sequence_length + pad_size+ query = query * (query.shape[-1] ** -0.5)++ v_beta = value * beta.unsqueeze(-1)+ k_beta = key * beta.unsqueeze(-1)+ query, key, value, k_beta, v_beta = [+ x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) for x in (query, key, value, k_beta, v_beta)+ ]+ g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)+ lower_mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)++ g = g.cumsum(dim=-1)+ decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()+ attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(lower_mask, 0)+ for idx in range(1, chunk_size):+ row = attn[..., idx, :idx].clone()+ sub = attn[..., :idx, :idx].clone()+ attn[..., idx, :idx] = row + (row.unsqueeze(-1) * sub).sum(-2)+ attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)+ value = attn @ v_beta+ k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))+ recurrent_state = (+ torch.zeros(batch_size, num_heads, k_head_dim, v_head_dim, device=value.device, dtype=value.dtype)+ if initial_state is None+ else initial_state.to(value)+ )+ outputs = torch.zeros_like(value)+ upper_mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=1)++ for idx in range(0, total_sequence_length // chunk_size):+ q_i = query[:, :, idx]+ k_i = key[:, :, idx]+ v_i = value[:, :, idx]+ attn = (q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, idx]).masked_fill_(upper_mask, 0)+ v_prime = k_cumdecay[:, :, idx] @ recurrent_state+ v_new = v_i - v_prime+ attn_inter = (q_i * g[:, :, idx, :, None].exp()) @ recurrent_state+ outputs[:, :, idx] = attn_inter + attn @ v_new+ recurrent_state = (+ recurrent_state * g[:, :, idx, -1, None, None].exp()+ + (k_i * (g[:, :, idx, -1, None] - g[:, :, idx]).exp()[..., None]).transpose(-1, -2) @ v_new+ )++ outputs = outputs.reshape(outputs.shape[0], outputs.shape[1], -1, outputs.shape[-1])+ outputs = outputs[:, :, :sequence_length]+ if not output_final_state:+ recurrent_state = None+ return outputs.transpose(1, 2).contiguous().to(initial_dtype), recurrent_state+++def torch_recurrent_gated_delta_rule(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ initial_state: torch.Tensor | None,+ output_final_state: bool,+) -> tuple[torch.Tensor, torch.Tensor | None]:+ initial_dtype = query.dtype+ query = l2norm(query, dim=-1)+ key = l2norm(key, dim=-1)+ query, key, value, beta, g = [+ x.transpose(1, 2).contiguous().to(torch.float32) for x in (query, key, value, beta, g)+ ]++ batch_size, num_heads, sequence_length, k_head_dim = key.shape+ v_head_dim = value.shape[-1]+ query = query * (query.shape[-1] ** -0.5)+ outputs = torch.zeros(batch_size, num_heads, sequence_length, v_head_dim, device=value.device, dtype=value.dtype)+ recurrent_state = (+ torch.zeros(batch_size, num_heads, k_head_dim, v_head_dim, device=value.device, dtype=value.dtype)+ if initial_state is None+ else initial_state.to(value)+ )++ for idx in range(sequence_length):+ q_t = query[:, :, idx]+ k_t = key[:, :, idx]+ v_t = value[:, :, idx]+ g_t = g[:, :, idx].exp().unsqueeze(-1).unsqueeze(-1)+ beta_t = beta[:, :, idx].unsqueeze(-1)+ recurrent_state = recurrent_state * g_t+ kv_mem = (recurrent_state * k_t.unsqueeze(-1)).sum(dim=-2)+ delta = (v_t - kv_mem) * beta_t+ recurrent_state = recurrent_state + k_t.unsqueeze(-1) * delta.unsqueeze(-2)+ outputs[:, :, idx] = (recurrent_state * q_t.unsqueeze(-1)).sum(dim=-2)++ if not output_final_state:+ recurrent_state = None+ return outputs.transpose(1, 2).contiguous().to(initial_dtype), recurrent_state+++class Qwen3_5DynamicCache:+ def __init__(self, config):+ self.layer_types = list(config.layer_types)+ self.transformer_layers = [idx for idx, layer_type in enumerate(self.layer_types) if layer_type == "full_attention"]+ self.last_linear_layer = len(self.layer_types) - 1 - self.layer_types[::-1].index("linear_attention")+ self.conv_states = [None for _ in range(int(config.num_hidden_layers))]+ self.recurrent_states = [None for _ in range(int(config.num_hidden_layers))]+ self.key_cache = [None for _ in range(int(config.num_hidden_layers))]+ self.value_cache = [None for _ in range(int(config.num_hidden_layers))]++ def update(+ self,+ key_states: torch.Tensor,+ value_states: torch.Tensor,+ layer_idx: int,+ ) -> tuple[torch.Tensor, torch.Tensor]:+ if self.key_cache[layer_idx] is None:+ self.key_cache[layer_idx] = key_states+ self.value_cache[layer_idx] = value_states+ else:+ self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=2)+ self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=2)+ return self.key_cache[layer_idx], self.value_cache[layer_idx]++ def get_seq_length(self, layer_idx: int | None = 0) -> int:+ if not self.transformer_layers:+ return 0+ layer_idx = self.transformer_layers[0] if layer_idx not in self.transformer_layers else layer_idx+ cache = self.key_cache[layer_idx]+ if cache is None:+ return 0+ return int(cache.shape[-2])++ @property+ def has_previous_state(self) -> bool:+ return self.conv_states[self.last_linear_layer] is not None+++class Qwen3_5TextRotaryEmbedding(nn.Module):+ def __init__(self, config):+ super().__init__()+ rope_parameters = getattr(config, "rope_parameters", None) or {}+ head_dim = int(getattr(config, "head_dim", config.hidden_size // config.num_attention_heads))+ rotary_dim = int(head_dim * float(rope_parameters.get("partial_rotary_factor", 1.0)))+ rope_theta = float(rope_parameters.get("rope_theta", 1000000))+ inv_freq = 1.0 / (rope_theta ** (torch.arange(0, rotary_dim, 2, dtype=torch.float32, device="cpu") / rotary_dim))+ self.register_buffer("inv_freq", inv_freq, persistent=False)+ self.mrope_section = list(rope_parameters.get("mrope_section", [11, 11, 10]))+ self.max_position_embeddings = int(getattr(config, "max_position_embeddings", 32768))+ self.rope_theta = rope_theta+ self.rotary_dim = rotary_dim++ def apply_interleaved_mrope(self, freqs: torch.Tensor) -> torch.Tensor:+ freqs_t = freqs[0]+ for dim, offset in enumerate((1, 2), start=1):+ length = self.mrope_section[dim] * 3+ idx = slice(offset, length, 3)+ freqs_t[..., idx] = freqs[dim, ..., idx]+ return freqs_t++ def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ if position_ids.ndim == 1:+ position_ids = position_ids.unsqueeze(0)+ if position_ids.ndim == 2:+ position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)+ max_position = self.max_position_embeddings+ if not (position_ids.is_cuda and torch.cuda.is_current_stream_capturing()):+ max_position = max(max_position, int(position_ids.max().item()) + 1)+ freq_cache = _build_mrope_freq_cache(+ position_ids.device,+ max_position=max_position,+ rotary_dim=self.rotary_dim,+ rope_theta=self.rope_theta,+ )+ freqs = torch.stack([freq_cache[position_ids[axis].long()] for axis in range(3)], dim=0)+ freqs = self.apply_interleaved_mrope(freqs)+ emb = torch.cat((freqs, freqs), dim=-1)+ return emb.cos().to(dtype=x.dtype), emb.sin().to(dtype=x.dtype)+++class Qwen3_5RMSNormGated(nn.Module):+ def __init__(self, dim: int, eps: float = 1e-6):+ super().__init__()+ self.eps = eps+ self.weight = nn.Parameter(torch.ones(dim))++ def forward(self, hidden_states: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:+ input_dtype = hidden_states.dtype+ hidden_states = hidden_states.float()+ hidden_states = hidden_states * torch.rsqrt(hidden_states.pow(2).mean(-1, keepdim=True) + self.eps)+ hidden_states = hidden_states * self.weight.float()+ hidden_states = hidden_states * F.silu(gate.float())+ return hidden_states.to(dtype=input_dtype)+++def _repeat_kv(hidden_states: torch.Tensor, num_repeats: int) -> torch.Tensor:+ if num_repeats == 1:+ return hidden_states+ return hidden_states.repeat_interleave(num_repeats, dim=1)+++def _flash_attention(+ query_states: torch.Tensor,+ key_states: torch.Tensor,+ value_states: torch.Tensor,+ query_lengths: list[int],+ key_lengths: list[int],+ scaling: float,+) -> torch.Tensor:+ if flash_attn_varlen_func is not None and query_states.is_cuda:+ q_chunks = [query_states[idx, : q_len] for idx, q_len in enumerate(query_lengths) if q_len > 0]+ k_chunks = [key_states[idx, : k_len] for idx, k_len in enumerate(key_lengths) if k_len > 0]+ v_chunks = [value_states[idx, : k_len] for idx, k_len in enumerate(key_lengths) if k_len > 0]+ q_flat = torch.cat(q_chunks, dim=0)+ k_flat = torch.cat(k_chunks, dim=0)+ v_flat = torch.cat(v_chunks, dim=0)+ cu_q = _build_cu_seqlens(query_lengths, query_states.device)+ cu_k = _build_cu_seqlens(key_lengths, query_states.device)+ out_flat = flash_attn_varlen_func(+ q_flat,+ k_flat,+ v_flat,+ cu_seqlens_q=cu_q,+ cu_seqlens_k=cu_k,+ max_seqlen_q=max(query_lengths),+ max_seqlen_k=max(key_lengths),+ softmax_scale=scaling,+ causal=True,+ )+ out = torch.zeros_like(query_states)+ offset = 0+ for idx, q_len in enumerate(query_lengths):+ if q_len <= 0:+ continue+ out[idx, :q_len] = out_flat[offset : offset + q_len]+ offset += q_len+ return out++ outputs = []+ for idx, q_len in enumerate(query_lengths):+ q = query_states[idx, :q_len].transpose(0, 1)+ k = key_states[idx, : key_lengths[idx]].transpose(0, 1)+ v = value_states[idx, : key_lengths[idx]].transpose(0, 1)+ k = _repeat_kv(k.unsqueeze(0), q.shape[0] // k.shape[0]).squeeze(0)+ v = _repeat_kv(v.unsqueeze(0), q.shape[0] // v.shape[0]).squeeze(0)+ out = F.scaled_dot_product_attention(+ q.unsqueeze(0),+ k.unsqueeze(0),+ v.unsqueeze(0),+ attn_mask=None,+ dropout_p=0.0,+ is_causal=True,+ scale=scaling,+ ).squeeze(0).transpose(0, 1)+ outputs.append(out)+ padded = torch.zeros_like(query_states)+ for idx, out in enumerate(outputs):+ padded[idx, : out.shape[0]] = out+ return padded+++class Qwen3_5Block(nn.Module):+ def __init__(self, config, layer_idx: int):+ super().__init__()+ self.layer_type = str(config.layer_types[layer_idx])+ self.attn_norm = RMSNorm(int(config.hidden_size), eps=float(config.rms_norm_eps))+ self.post_attention_norm = RMSNorm(int(config.hidden_size), eps=float(config.rms_norm_eps))+ self.ffn_gate = ColumnParallelLinear(int(config.hidden_size), int(config.intermediate_size), bias=False)+ self.ffn_up = ColumnParallelLinear(int(config.hidden_size), int(config.intermediate_size), bias=False)+ self.ffn_down = RowParallelLinear(int(config.intermediate_size), int(config.hidden_size), bias=False)+ self.mlp_act_fn = SiluAndMul()++ if self.layer_type == "full_attention":+ tp_size = _get_tp_size()+ hidden_size = int(config.hidden_size)+ total_num_heads = int(config.num_attention_heads)+ total_num_kv_heads = int(config.num_key_value_heads)+ assert total_num_heads % tp_size == 0+ assert total_num_kv_heads % tp_size == 0+ self.num_heads = total_num_heads // tp_size+ self.num_kv_heads = total_num_kv_heads // tp_size+ self.head_dim = int(getattr(config, "head_dim", hidden_size // total_num_heads))+ self.num_key_value_groups = self.num_heads // self.num_kv_heads+ self.scaling = self.head_dim**-0.5+ self.attn_q = ColumnParallelLinear(+ hidden_size,+ total_num_heads * self.head_dim * 2,+ bias=bool(config.attention_bias),+ )+ self.attn_k = ColumnParallelLinear(+ hidden_size,+ total_num_kv_heads * self.head_dim,+ bias=bool(config.attention_bias),+ )+ self.attn_v = ColumnParallelLinear(+ hidden_size,+ total_num_kv_heads * self.head_dim,+ bias=bool(config.attention_bias),+ )+ self.attn_output = RowParallelLinear(+ total_num_heads * self.head_dim,+ hidden_size,+ bias=bool(config.attention_bias),+ )+ self.attn = Attention(self.num_heads, self.head_dim, self.scaling, self.num_kv_heads)+ self.attn_q_norm = RMSNorm(self.head_dim, eps=float(config.rms_norm_eps))+ self.attn_k_norm = RMSNorm(self.head_dim, eps=float(config.rms_norm_eps))+ else:+ self.num_v_heads = int(config.linear_num_value_heads)+ self.num_k_heads = int(config.linear_num_key_heads)+ self.head_k_dim = int(config.linear_key_head_dim)+ self.head_v_dim = int(config.linear_value_head_dim)+ self.key_dim = self.head_k_dim * self.num_k_heads+ self.value_dim = self.head_v_dim * self.num_v_heads+ self.conv_kernel_size = int(config.linear_conv_kernel_dim)+ hidden_size = int(config.hidden_size)+ self.attn_qkv = ColumnParallelLinear(hidden_size, self.key_dim * 2 + self.value_dim, bias=False)+ self.attn_gate = ColumnParallelLinear(hidden_size, self.value_dim, bias=False)+ self.ssm_alpha = ColumnParallelLinear(hidden_size, self.num_v_heads, bias=False)+ self.ssm_beta = ColumnParallelLinear(hidden_size, self.num_v_heads, bias=False)+ self.ssm_dt = nn.Parameter(torch.zeros(self.num_v_heads))+ self.ssm_a = nn.Parameter(-torch.ones(self.num_v_heads))+ if ShortConvolution is not None:+ self.ssm_conv1d = ShortConvolution(+ hidden_size=self.key_dim * 2 + self.value_dim,+ kernel_size=self.conv_kernel_size,+ bias=False,+ activation="silu",+ )+ self._use_short_convolution = True+ else:+ self.ssm_conv1d = nn.Conv1d(+ in_channels=self.key_dim * 2 + self.value_dim,+ out_channels=self.key_dim * 2 + self.value_dim,+ kernel_size=self.conv_kernel_size,+ bias=False,+ groups=self.key_dim * 2 + self.value_dim,+ padding=self.conv_kernel_size - 1,+ )+ self._use_short_convolution = False+ if FusedRMSNormGated is not None:+ self.ssm_norm = FusedRMSNormGated(self.head_v_dim, eps=float(config.rms_norm_eps))+ self._use_fused_rmsnorm_gated = True+ else:+ self.ssm_norm = Qwen3_5RMSNormGated(self.head_v_dim, eps=float(config.rms_norm_eps))+ self._use_fused_rmsnorm_gated = False+ self.ssm_out = RowParallelLinear(self.value_dim, hidden_size, bias=False)+ self.conv_state_buffer = torch.empty(0)+ self.recurrent_state_buffer = torch.empty(0)+ self._gguf_interleave_ssm_ab = False++ def prepare_sequence_state(self, max_batch_size: int, device: torch.device, dtype: torch.dtype):+ if self.layer_type != "linear_attention":+ return+ conv_shape = (int(max_batch_size), self.key_dim * 2 + self.value_dim, self.conv_kernel_size)+ recurrent_shape = (int(max_batch_size), self.num_v_heads, self.head_k_dim, self.head_v_dim)+ with torch.inference_mode():+ if (+ self.conv_state_buffer.numel() == 0+ or tuple(self.conv_state_buffer.shape) != conv_shape+ or self.conv_state_buffer.device != device+ or self.conv_state_buffer.dtype != dtype+ ):+ self.conv_state_buffer = torch.zeros(conv_shape, device=device, dtype=dtype)+ else:+ self.conv_state_buffer.zero_()+ if (+ self.recurrent_state_buffer.numel() == 0+ or tuple(self.recurrent_state_buffer.shape) != recurrent_shape+ or self.recurrent_state_buffer.device != device+ or self.recurrent_state_buffer.dtype != dtype+ ):+ self.recurrent_state_buffer = torch.zeros(recurrent_shape, device=device, dtype=dtype)+ else:+ self.recurrent_state_buffer.zero_()++ def reset_sequence_state(self):+ if self.layer_type != "linear_attention":+ return+ with torch.inference_mode():+ if self.conv_state_buffer.numel() > 0:+ self.conv_state_buffer.zero_()+ if self.recurrent_state_buffer.numel() > 0:+ self.recurrent_state_buffer.zero_()++ def release_sequence_state(self):+ if self.layer_type != "linear_attention":+ return+ with torch.inference_mode():+ self.conv_state_buffer = torch.empty(0)+ self.recurrent_state_buffer = torch.empty(0)++ def _get_runtime_conv_state(self, batch_size: int, hidden_states: torch.Tensor) -> torch.Tensor:+ if (+ self.conv_state_buffer.numel() == 0+ or self.conv_state_buffer.shape[0] < int(batch_size)+ or self.conv_state_buffer.device != hidden_states.device+ or self.conv_state_buffer.dtype != hidden_states.dtype+ ):+ self.prepare_sequence_state(batch_size, hidden_states.device, hidden_states.dtype)+ return self.conv_state_buffer[:batch_size]++ def _get_runtime_recurrent_state(self, batch_size: int, hidden_states: torch.Tensor) -> torch.Tensor:+ if (+ self.recurrent_state_buffer.numel() == 0+ or self.recurrent_state_buffer.shape[0] < int(batch_size)+ or self.recurrent_state_buffer.device != hidden_states.device+ or self.recurrent_state_buffer.dtype != hidden_states.dtype+ ):+ self.prepare_sequence_state(batch_size, hidden_states.device, hidden_states.dtype)+ return self.recurrent_state_buffer[:batch_size]++ def _forward_full_attention(+ self,+ hidden_states: torch.Tensor,+ position_embeddings: tuple[torch.Tensor, torch.Tensor],+ layer_idx: int,+ past_key_values: Qwen3_5DynamicCache | None,+ ) -> torch.Tensor:+ batch_size, seq_len, _ = hidden_states.shape+ q_and_gate = self.attn_q(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim * 2)+ query_states, gate = torch.chunk(q_and_gate, 2, dim=-1)+ gate = gate.reshape(batch_size, seq_len, -1)++ query_states = self.attn_q_norm(query_states)+ key_states = self.attn_k_norm(self.attn_k(hidden_states).view(batch_size, seq_len, self.num_kv_heads, self.head_dim))+ value_states = self.attn_v(hidden_states).view(batch_size, seq_len, self.num_kv_heads, self.head_dim)++ cos, sin = position_embeddings+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)++ if past_key_values is not None:+ key_states, value_states = past_key_values.update(+ key_states.transpose(1, 2),+ value_states.transpose(1, 2),+ layer_idx,+ )+ key_states = key_states.transpose(1, 2)+ value_states = value_states.transpose(1, 2)+ attn_output = _flash_attention(+ query_states,+ key_states,+ value_states,+ query_lengths=[int(seq_len)] * batch_size,+ key_lengths=[int(key_states.shape[1])] * batch_size,+ scaling=self.scaling,+ ).reshape(batch_size, seq_len, -1)+ else:+ attn_output = self.attn(+ query_states.reshape(-1, self.num_heads, self.head_dim),+ key_states.reshape(-1, self.num_kv_heads, self.head_dim),+ value_states.reshape(-1, self.num_kv_heads, self.head_dim),+ ).reshape(batch_size, seq_len, -1)+ attn_output = attn_output * torch.sigmoid(gate)+ return self.attn_output(attn_output)++ def _forward_linear_attention(+ self,+ hidden_states: torch.Tensor,+ layer_idx: int,+ cache_params: Qwen3_5DynamicCache | None,+ attention_mask: torch.Tensor | None,+ ) -> torch.Tensor:+ if attention_mask is not None and attention_mask.shape[1] > 1 and attention_mask.shape[0] > 1:+ hidden_states = (hidden_states * attention_mask[:, :, None]).to(hidden_states.dtype)++ batch_size, seq_len, _ = hidden_states.shape+ context = get_context()+ if cache_params is not None:+ conv_state = cache_params.conv_states[layer_idx]+ recurrent_state = cache_params.recurrent_states[layer_idx]+ has_previous_state = cache_params.has_previous_state+ else:+ conv_state = self._get_runtime_conv_state(batch_size, hidden_states)+ recurrent_state = self._get_runtime_recurrent_state(batch_size, hidden_states)+ has_previous_state = not context.is_prefill+ use_precomputed_states = has_previous_state and seq_len == 1++ mixed_qkv_input = self.attn_qkv(hidden_states)+ z = self.attn_gate(hidden_states).reshape(batch_size, seq_len, -1, self.head_v_dim)+ b = self.ssm_beta(hidden_states)+ a = self.ssm_alpha(hidden_states)+ if self._gguf_interleave_ssm_ab:+ b = _interleave_last_dim_halves(b)+ a = _interleave_last_dim_halves(a)++ use_short_convolution = self._use_short_convolution and ShortConvolution is not None and isinstance(self.ssm_conv1d, ShortConvolution)++ if use_short_convolution:+ short_conv_cache = conv_state if has_previous_state and conv_state is not None else None+ mixed_qkv, last_conv_state = self.ssm_conv1d(+ mixed_qkv_input,+ cache=short_conv_cache,+ output_final_state=True,+ )+ else:+ mixed_qkv = mixed_qkv_input.transpose(1, 2)+ use_fast_causal_conv = causal_conv1d_fn is not None and causal_conv1d_update is not None+ if use_precomputed_states:+ if use_fast_causal_conv:+ conv_kernel = self.ssm_conv1d.weight.squeeze(1)+ mixed_qkv = causal_conv1d_update(+ mixed_qkv,+ conv_state,+ conv_kernel,+ self.ssm_conv1d.bias,+ "silu",+ )+ else:+ conv_kernel = self.ssm_conv1d.weight.reshape(+ self.ssm_conv1d.weight.shape[0],+ self.ssm_conv1d.weight.shape[-1],+ )+ mixed_qkv = torch_causal_conv1d_update(+ mixed_qkv,+ conv_state,+ conv_kernel,+ self.ssm_conv1d.bias,+ )+ else:+ if cache_params is not None:+ if mixed_qkv.shape[-1] >= self.conv_kernel_size:+ cache_params.conv_states[layer_idx] = mixed_qkv[:, :, -self.conv_kernel_size :].contiguous()+ else:+ cache_params.conv_states[layer_idx] = F.pad(+ mixed_qkv,+ (self.conv_kernel_size - mixed_qkv.shape[-1], 0),+ )+ else:+ if mixed_qkv.shape[-1] >= self.conv_kernel_size:+ conv_state.copy_(mixed_qkv[:, :, -self.conv_kernel_size :].contiguous())+ else:+ conv_state.copy_(+ F.pad(+ mixed_qkv,+ (self.conv_kernel_size - mixed_qkv.shape[-1], 0),+ )+ )+ conv_input = mixed_qkv.to(self.ssm_conv1d.weight.dtype)+ if use_fast_causal_conv:+ mixed_qkv = causal_conv1d_fn(+ x=conv_input,+ weight=self.ssm_conv1d.weight.squeeze(1),+ bias=self.ssm_conv1d.bias,+ activation="silu",+ seq_idx=None,+ )+ else:+ mixed_qkv = F.silu(self.ssm_conv1d(conv_input)[:, :, :seq_len])+ mixed_qkv = mixed_qkv.to(hidden_states.dtype)+ mixed_qkv = mixed_qkv.transpose(1, 2)++ if use_short_convolution:+ if cache_params is not None:+ cache_params.conv_states[layer_idx] = last_conv_state+ elif last_conv_state is not conv_state:+ conv_state.copy_(last_conv_state)++ query, key, value = torch.split(mixed_qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1)+ query = query.reshape(batch_size, seq_len, -1, self.head_k_dim)+ key = key.reshape(batch_size, seq_len, -1, self.head_k_dim)+ value = value.reshape(batch_size, seq_len, -1, self.head_v_dim)++ beta = b.sigmoid()+ g = self.ssm_a.float() * F.softplus(a.float() + self.ssm_dt)+ if self.num_v_heads // self.num_k_heads > 1:+ repeat_factor = self.num_v_heads // self.num_k_heads+ query = query.repeat_interleave(repeat_factor, dim=2)+ key = key.repeat_interleave(repeat_factor, dim=2)++ if use_precomputed_states:+ if fast_recurrent_gated_delta_rule is not None and hidden_states.is_cuda:+ core_attn_out, last_recurrent_state = fast_recurrent_gated_delta_rule(+ query,+ key,+ value,+ g=g,+ beta=beta,+ initial_state=recurrent_state,+ output_final_state=True,+ use_qk_l2norm_in_kernel=True,+ )+ else:+ core_attn_out, last_recurrent_state = torch_recurrent_gated_delta_rule(+ query,+ key,+ value,+ g=g,+ beta=beta,+ initial_state=recurrent_state,+ output_final_state=True,+ )+ else:+ if fast_chunk_gated_delta_rule is not None and hidden_states.is_cuda:+ core_attn_out, last_recurrent_state = fast_chunk_gated_delta_rule(+ query,+ key,+ value,+ g=g,+ beta=beta,+ initial_state=None,+ output_final_state=True,+ use_qk_l2norm_in_kernel=True,+ )+ else:+ core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule(+ query,+ key,+ value,+ g=g,+ beta=beta,+ initial_state=None,+ output_final_state=True,+ )++ if cache_params is not None:+ cache_params.recurrent_states[layer_idx] = last_recurrent_state+ else:+ recurrent_state.copy_(last_recurrent_state)++ core_attn_out = self.ssm_norm(+ core_attn_out.reshape(-1, self.head_v_dim),+ z.reshape(-1, self.head_v_dim),+ )+ core_attn_out = core_attn_out.reshape(batch_size, seq_len, -1)+ return self.ssm_out(core_attn_out)++ def forward(+ self,+ hidden_states: torch.Tensor,+ position_embeddings: tuple[torch.Tensor, torch.Tensor],+ residual: torch.Tensor | None,+ layer_idx: int,+ attention_mask: torch.Tensor | None = None,+ past_key_values: Qwen3_5DynamicCache | None = None,+ ) -> tuple[torch.Tensor, torch.Tensor]:+ if residual is None:+ hidden_states, residual = self.attn_norm(hidden_states), hidden_states+ else:+ hidden_states, residual = self.attn_norm(hidden_states, residual)++ if self.layer_type == "full_attention":+ hidden_states = self._forward_full_attention(hidden_states, position_embeddings, layer_idx, past_key_values)+ else:+ hidden_states = self._forward_linear_attention(hidden_states, layer_idx, past_key_values, attention_mask)++ hidden_states, residual = self.post_attention_norm(hidden_states, residual)+ hidden_states = self.ffn_down(self.mlp_act_fn(torch.cat((self.ffn_gate(hidden_states), self.ffn_up(hidden_states)), dim=-1)))+ return hidden_states, residual+++class Qwen3_5ForCausalLM(nn.Module):+ def __init__(self, config) -> None:+ super().__init__()+ self.config = config+ self.token_embd = nn.Embedding(int(config.vocab_size), int(config.hidden_size))+ self.rotary_emb = Qwen3_5TextRotaryEmbedding(config)+ self.blk = nn.ModuleList([Qwen3_5Block(config, idx) for idx in range(int(config.num_hidden_layers))])+ self.output_norm = RMSNorm(int(config.hidden_size), eps=float(config.rms_norm_eps))+ self.output = nn.Linear(int(config.hidden_size), int(config.vocab_size), bias=False)++ @property+ def device(self) -> torch.device:+ return next(self.parameters()).device++ @property+ def embed_tokens(self):+ return self.token_embd++ @property+ def layers(self):+ return self.blk++ @property+ def norm(self):+ return self.output_norm++ @property+ def lm_head(self):+ return self.output++ def forward(+ self,+ input_ids: torch.Tensor | None = None,+ positions: torch.Tensor | None = None,+ position_ids: torch.Tensor | None = None,+ inputs_embeds: torch.Tensor | None = None,+ attention_mask: torch.Tensor | None = None,+ past_key_values: Qwen3_5DynamicCache | None = None,+ use_cache: bool | None = None,+ **_kwargs,+ ) -> torch.Tensor:+ del use_cache+ if (input_ids is None) == (inputs_embeds is None):+ raise ValueError("Specify exactly one of input_ids or inputs_embeds.")++ if position_ids is not None:+ positions = position_ids++ if input_ids is not None and input_ids.ndim == 1:+ input_ids = input_ids.unsqueeze(0)+ if positions is None:+ seq_len = inputs_embeds.shape[1] if inputs_embeds is not None else input_ids.shape[1]+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0+ positions = torch.arange(+ past_seen_tokens,+ past_seen_tokens + seq_len,+ device=(inputs_embeds.device if inputs_embeds is not None else input_ids.device),+ dtype=torch.long,+ ).unsqueeze(0)+ elif positions.ndim == 1:+ positions = positions.unsqueeze(0)+ elif positions.ndim == 3 and positions.shape[0] == 4:+ positions = positions[1:]++ hidden_states = self.token_embd(input_ids) if inputs_embeds is None else inputs_embeds+ position_embeddings = self.rotary_emb(hidden_states, positions)+ residual = None+ for layer_idx, block in enumerate(self.blk):+ hidden_states, residual = block(+ hidden_states,+ position_embeddings,+ residual,+ layer_idx=layer_idx,+ attention_mask=attention_mask,+ past_key_values=past_key_values,+ )+ hidden_states, _ = self.output_norm(hidden_states, residual)+ return hidden_states++ def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:+ context = get_context()+ if hidden_states.ndim == 3:+ if context.is_prefill and context.cu_seqlens_q is not None and context.cu_seqlens_q.numel() > 1:+ num_seqs = int(context.cu_seqlens_q.numel() - 1)+ if hidden_states.shape[0] == num_seqs:+ lengths = context.cu_seqlens_q[1:] - context.cu_seqlens_q[:-1]+ row_indices = torch.arange(num_seqs, device=hidden_states.device)+ token_indices = lengths.to(device=hidden_states.device, dtype=torch.long) - 1+ hidden_states = hidden_states[row_indices, token_indices]+ elif hidden_states.shape[0] == 1 and num_seqs == 1:+ last_index = int(context.cu_seqlens_q[-1].item()) - 1+ hidden_states = hidden_states[:, last_index, :]+ elif hidden_states.shape[-2] == 1:+ hidden_states = hidden_states[:, 0, :]+ return self.output(hidden_states)+++__all__ = ["Qwen3_5DynamicCache", "Qwen3_5ForCausalLM", "clear_qwen35_runtime_caches"]diff --git a/shared/llm_engines/nanovllm/vllm_support.py b/shared/llm_engines/nanovllm/vllm_support.pyindex c5ce67a..b3357b2 100644--- a/shared/llm_engines/nanovllm/vllm_support.py+++ b/shared/llm_engines/nanovllm/vllm_support.py@@ -153,13 +153,29 @@ def resolve_lm_decoder_engine(requested_engine, engines_available = []):return "legacy"+def _clear_inductor_cuda_pools():+ try:+ from torch._inductor import cudagraph_trees as cgt+ except Exception:+ return++ clear_cublass_cache = getattr(cgt, "clear_cublass_cache", None)+ if callable(clear_cublass_cache):+ try:+ clear_cublass_cache()+ except Exception:+ pass++class NanoVllmTextEngine:keep_loaded_for_phase2 = True- def __init__(self, model, model_path: str, tokenizer):+ def __init__(self, model, model_path: str, tokenizer, enforce_eager: bool = False):self.model = modelself.model_path = model_pathself.tokenizer = tokenizer+ self.enforce_eager = bool(enforce_eager)+ self.hf_config = getattr(model, "config", None)self._llm = Noneself._sampling_params_cls = Noneself._max_model_len_hint = None@@ -223,11 +239,12 @@ class NanoVllmTextEngine:max_num_batched_tokens = self._max_num_batched_tokens_hint or (max_model_len * max_num_seqs)self._llm = LLM(model=self.model_path,- enforce_eager=False,+ enforce_eager=self.enforce_eager,tensor_parallel_size=1,max_model_len=max_model_len,max_num_seqs=max_num_seqs,max_num_batched_tokens=max_num_batched_tokens,+ hf_config=self.hf_config,tokenizer=self.tokenizer,model_object=self.model,)@@ -268,6 +285,10 @@ class NanoVllmTextEngine:except Exception:passself._sampling_params_cls = None+ try:+ _clear_inductor_cuda_pools()+ except Exception:+ passdef __del__(self):self.close()@@ -408,3 +429,81 @@ class NanoVllmTextEngine:)return {"token_ids": token_ids, "text": text}++ def generate_embedded(+ self,+ prompt_token_ids: list[int],+ prompt_embeds,+ prompt_position_ids,+ max_tokens: int,+ temperature: Optional[float],+ top_p: Optional[float],+ top_k: Optional[int],+ cfg_scale: float,+ seed: Optional[int],+ use_tqdm: bool = True,+ release_vram_after: bool = True,+ ignore_eos: bool = False,+ position_offset: int = 0,+ ):+ req_model_len, req_num_seqs, req_num_batched = self._compute_runtime_hints(+ prompt_len=len(prompt_token_ids),+ max_tokens=max_tokens,+ cfg_scale=cfg_scale,+ )+ self._ensure_runtime_capacity(req_model_len, req_num_seqs, req_num_batched)+ self._ensure_llm()+ if self._llm is None:+ return None+ try:+ self._llm.reset()+ except Exception:+ pass++ seed_value = None+ if seed is not None:+ try:+ seed_value = int(seed)+ except Exception:+ seed_value = None+ if seed_value is not None and seed_value < 0:+ seed_value = None++ temp = temperature if temperature is not None and temperature > 0 else 1e-5+ sampling_params = self._sampling_params_cls(+ temperature=temp,+ max_tokens=max_tokens,+ cfg_scale=max(cfg_scale, 1.0),+ top_k=top_k if top_k is not None and top_k > 0 else None,+ top_p=top_p if top_p is not None and 0.0 < top_p < 1.0 else None,+ ignore_eos=bool(ignore_eos),+ seed=seed_value,+ )++ text = ""+ token_ids: list[int] = []+ try:+ outputs = self._llm.generate_embedded(+ prompts=[prompt_token_ids],+ prompt_embeds=[prompt_embeds],+ prompt_position_ids=[prompt_position_ids],+ position_offsets=[int(position_offset)],+ sampling_params=sampling_params,+ use_tqdm=use_tqdm,+ )+ if outputs:+ text, token_ids = self._extract_text_and_tokens(outputs[0])+ if (not text) and token_ids:+ try:+ text = self.tokenizer.decode(token_ids, skip_special_tokens=False)+ except Exception:+ text = ""+ self._last_failure_reason = ""+ except Exception as exc:+ self._last_failure_reason = str(exc)+ raise+ finally:+ if release_vram_after:+ self.release_runtime_allocations()++ return {"token_ids": token_ids, "text": text}diff --git a/shared/prompt_enhancer/__init__.py b/shared/prompt_enhancer/__init__.pyindex 198074e..7a56561 100644--- a/shared/prompt_enhancer/__init__.py+++ b/shared/prompt_enhancer/__init__.py@@ -1,3 +1,44 @@-from .loader import load_florence2+from importlib import import_module-__all__ = ["load_florence2"]++_EXPORTS = {+ "load_florence2": ("shared.prompt_enhancer.loader", "load_florence2"),+ "ensure_prompt_enhancer_assets": ("shared.prompt_enhancer.loader", "ensure_prompt_enhancer_assets"),+ "load_prompt_enhancer_runtime": ("shared.prompt_enhancer.loader", "load_prompt_enhancer_runtime"),+ "unload_prompt_enhancer_models": ("shared.prompt_enhancer.loader", "unload_prompt_enhancer_models"),+ "load_qwen35_prompt_enhancer": ("shared.prompt_enhancer.qwen35_text", "load_qwen35_prompt_enhancer"),+ "load_qwen35_text_prompt_enhancer": ("shared.prompt_enhancer.qwen35_text", "load_qwen35_text_prompt_enhancer"),+ "QWEN35_VARIANT_9B": ("shared.prompt_enhancer.qwen35_vl", "QWEN35_VARIANT_9B"),+ "QWEN35_VARIANT_4B": ("shared.prompt_enhancer.qwen35_vl", "QWEN35_VARIANT_4B"),+ "QWEN35_ABLITERATED_TEXT_REQUIRED_FILES": ("shared.prompt_enhancer.qwen35_vl", "QWEN35_ABLITERATED_TEXT_REQUIRED_FILES"),+ "enhancer_quantization_GGUF": ("shared.prompt_enhancer.qwen35_vl", "enhancer_quantization_GGUF"),+ "enhancer_quantization_QUANTO_INT8": ("shared.prompt_enhancer.qwen35_vl", "enhancer_quantization_QUANTO_INT8"),+ "enhancer_quantization_SAFETENSORS": ("shared.prompt_enhancer.qwen35_vl", "enhancer_quantization_SAFETENSORS"),+ "QWEN35_TEXT_GGUF_FILENAME": ("shared.prompt_enhancer.qwen35_vl", "QWEN35_TEXT_GGUF_FILENAME"),+ "QWEN35_VISION_FILENAME": ("shared.prompt_enhancer.qwen35_vl", "QWEN35_VISION_FILENAME"),+ "UPSTREAM_MODELING_FILENAME": ("shared.prompt_enhancer.qwen35_vl", "UPSTREAM_MODELING_FILENAME"),+ "get_qwen35_prompt_enhancer_variant": ("shared.prompt_enhancer.qwen35_vl", "get_qwen35_prompt_enhancer_variant"),+ "get_qwen35_assets_dir_name": ("shared.prompt_enhancer.qwen35_vl", "get_qwen35_assets_dir_name"),+ "get_qwen35_modeling_path": ("shared.prompt_enhancer.qwen35_vl", "get_qwen35_modeling_path"),+ "get_qwen35_variant_spec": ("shared.prompt_enhancer.qwen35_vl", "get_qwen35_variant_spec"),+ "ensure_qwen35_prompt_enhancer_assets": ("shared.prompt_enhancer.qwen35_vl", "ensure_qwen35_prompt_enhancer_assets"),+ "get_qwen35_text_gguf_path": ("shared.prompt_enhancer.qwen35_vl", "get_qwen35_text_gguf_path"),+ "load_qwen35_vl_prompt_enhancer": ("shared.prompt_enhancer.qwen35_vl", "load_qwen35_vl_prompt_enhancer"),+}++__all__ = list(_EXPORTS.keys())+++def __getattr__(name):+ target = _EXPORTS.get(name)+ if target is None:+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")+ module_name, attr_name = target+ module = import_module(module_name)+ value = getattr(module, attr_name)+ globals()[name] = value+ return value+++def __dir__():+ return sorted(set(globals()) | set(__all__))diff --git a/shared/prompt_enhancer/loader.py b/shared/prompt_enhancer/loader.pyindex 2bd5308..5b98756 100644--- a/shared/prompt_enhancer/loader.py+++ b/shared/prompt_enhancer/loader.py@@ -1,15 +1,174 @@from __future__ import annotations+from dataclasses import dataclass, fieldfrom pathlib import Path-from typing import Tuple+from typing import Any, Tupleimport torch+from mmgp import offloadfrom safetensors import safe_open+from shared.utils import files_locator as flfrom .florence2 import Florence2Config, Florence2ForConditionalGeneration, Florence2Processorfrom .florence2.image_processing_florence2 import Florence2ImageProcessorLite-from transformers import BartTokenizer, BartTokenizerFast+from transformers import AutoTokenizer, BartTokenizer, BartTokenizerFast+++FLORENCE2_FOLDER = "Florence2"+LLAMA32_FOLDER = "Llama3_2"+LLAMAJOY_FOLDER = "llama-joycaption-beta-one-hf-llava"+PROMPT_ENHANCER_REPO = "DeepBeepMeep/LTX_Video"+++@dataclass(slots=True)+class PromptEnhancerRuntime:+ image_caption_model: Any = None+ image_caption_processor: Any = None+ llm_model: Any = None+ llm_tokenizer: Any = None+ pipe_models: dict[str, Any] = field(default_factory=dict)+ budgets: dict[str, int] = field(default_factory=dict)+++def ensure_prompt_enhancer_assets(process_files_def, enhancer_enabled: int, qwen_backend: str = "quanto_int8"):+ enhancer_enabled = int(enhancer_enabled)+ if enhancer_enabled == 1:+ process_files_def(+ repoId=PROMPT_ENHANCER_REPO,+ sourceFolderList=[FLORENCE2_FOLDER, LLAMA32_FOLDER],+ fileList=[+ ["config.json", "configuration_florence2.py", "model.safetensors", "preprocessor_config.json", "tokenizer.json", "tokenizer_config.json"],+ ["config.json", "generation_config.json", "Llama3_2_quanto_bf16_int8.safetensors", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json"],+ ],+ )+ return+ if enhancer_enabled == 2:+ process_files_def(+ repoId=PROMPT_ENHANCER_REPO,+ sourceFolderList=[FLORENCE2_FOLDER, LLAMAJOY_FOLDER],+ fileList=[+ ["config.json", "configuration_florence2.py", "model.safetensors", "preprocessor_config.json", "tokenizer.json", "tokenizer_config.json"],+ ["config.json", "llama_config.json", "llama_joycaption_quanto_bf16_int8.safetensors", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json"],+ ],+ )+ return+ if enhancer_enabled in (3, 4):+ from .qwen35_vl import ensure_qwen35_prompt_enhancer_assets, get_qwen35_prompt_enhancer_variant++ ensure_qwen35_prompt_enhancer_assets(process_files_def, backend=qwen_backend, variant=get_qwen35_prompt_enhancer_variant(enhancer_enabled))+++def unload_prompt_enhancer_models(*models):+ seen = set()+ for model in models:+ if model is None:+ continue+ model_id = id(model)+ if model_id in seen:+ continue+ seen.add(model_id)+ unload = getattr(model, "unload", None)+ if callable(unload):+ unload()+++def _set_pad_token_from_tokenizer(model, tokenizer):+ model.generation_config.pad_token = tokenizer.eos_token+ if model.generation_config.pad_token_id is None:+ eos_token_id = model.generation_config.eos_token_id+ model.generation_config.pad_token_id = eos_token_id[0] if isinstance(eos_token_id, list) else eos_token_id+++def _load_llama32_prompt_enhancer():+ llm_model = offload.fast_load_transformers_model(+ fl.locate_file(f"{LLAMA32_FOLDER}/Llama3_2_quanto_bf16_int8.safetensors"),+ defaultConfigPath=fl.locate_file(f"{LLAMA32_FOLDER}/config.json", error_if_none=False),+ configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},+ writable_tensors=False,+ )+ llm_model._validate_model_kwargs = lambda *_args, **_kwargs: None+ llm_model._offload_hooks = ["generate"]+ llm_tokenizer = AutoTokenizer.from_pretrained(fl.locate_folder(LLAMA32_FOLDER))+ _set_pad_token_from_tokenizer(llm_model, llm_tokenizer)+ llm_model.eval()+ return llm_model, llm_tokenizer, 5000+++def _load_joycaption_prompt_enhancer():+ def preprocess_sd(sd, quant_map=None, tied_map=None):+ rules = {"model.language_model": "model", "model.vision_tower": None, "model.multi_modal_projector": None}+ return tuple(offload.map_state_dict([sd, quant_map, tied_map], rules))++ llm_model = offload.fast_load_transformers_model(+ fl.locate_file(f"{LLAMAJOY_FOLDER}/llama_joycaption_quanto_bf16_int8.safetensors"),+ forcedConfigPath=fl.locate_file(f"{LLAMAJOY_FOLDER}/llama_config.json", error_if_none=False),+ configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},+ preprocess_sd=preprocess_sd,+ writable_tensors=False,+ )+ llm_tokenizer = AutoTokenizer.from_pretrained(fl.locate_folder(LLAMAJOY_FOLDER))+ _set_pad_token_from_tokenizer(llm_model, llm_tokenizer)+ llm_model.eval()+ return llm_model, llm_tokenizer, 10000+++def load_prompt_enhancer_runtime(process_files_def, enhancer_enabled: int, lm_decoder_engine: str = "", qwen_backend: str = "quanto_int8") -> PromptEnhancerRuntime:+ enhancer_enabled = int(enhancer_enabled)+ runtime = PromptEnhancerRuntime()+ if enhancer_enabled <= 0:+ return runtime++ ensure_prompt_enhancer_assets(process_files_def, enhancer_enabled=enhancer_enabled, qwen_backend=qwen_backend)++ if enhancer_enabled in (3, 4):+ from .qwen35_text import load_qwen35_text_prompt_enhancer+ from .qwen35_vl import (+ enhancer_quantization_QUANTO_INT8,+ get_qwen35_assets_dir_name,+ get_qwen35_prompt_enhancer_variant,+ load_qwen35_vl_prompt_enhancer,+ )++ backend = qwen_backend or enhancer_quantization_QUANTO_INT8+ qwen35_variant = get_qwen35_prompt_enhancer_variant(enhancer_enabled)+ assets_dir_name = get_qwen35_assets_dir_name(qwen35_variant)+ assets_dir = fl.locate_folder(assets_dir_name, error_if_none=False) or fl.get_download_location(assets_dir_name)+ runtime.llm_model = load_qwen35_text_prompt_enhancer(+ assets_dir=assets_dir,+ backend=backend,+ attn_implementation="sdpa",+ requested_lm_engine=lm_decoder_engine,+ variant=qwen35_variant,+ )+ runtime.llm_tokenizer = getattr(runtime.llm_model, "_prompt_enhancer_tokenizer", None)+ runtime.llm_model.eval()+ runtime.image_caption_model, vision_tower_model = load_qwen35_vl_prompt_enhancer(+ assets_dir=assets_dir,+ attn_implementation="sdpa",+ text_model=runtime.llm_model,+ backend=backend,+ variant=qwen35_variant,+ )+ runtime.image_caption_processor = getattr(runtime.image_caption_model, "_prompt_enhancer_processor", None)+ runtime.image_caption_model.eval()+ runtime.pipe_models["prompt_enhancer_image_caption_vision_tower_model"] = vision_tower_model+ runtime.pipe_models["prompt_enhancer_llm_model"] = runtime.llm_model+ runtime.budgets["prompt_enhancer_image_caption_vision_tower_model"] = 3000+ runtime.budgets["prompt_enhancer_llm_model"] = 10000+ return runtime++ runtime.image_caption_model, runtime.image_caption_processor = load_florence2(fl.locate_folder(FLORENCE2_FOLDER), attn_implementation="sdpa")+ runtime.image_caption_model._model_dtype = torch.float+ runtime.image_caption_model.eval()+ runtime.pipe_models["prompt_enhancer_image_caption_model"] = runtime.image_caption_model+ if enhancer_enabled == 1:+ runtime.llm_model, runtime.llm_tokenizer, budget = _load_llama32_prompt_enhancer()+ else:+ runtime.llm_model, runtime.llm_tokenizer, budget = _load_joycaption_prompt_enhancer()+ runtime.pipe_models["prompt_enhancer_llm_model"] = runtime.llm_model+ runtime.budgets["prompt_enhancer_llm_model"] = budget+ return runtimedef _load_state_dict(weights_path: Path) -> dict:diff --git a/shared/prompt_enhancer/prompt_enhance_utils.py b/shared/prompt_enhancer/prompt_enhance_utils.pyindex 406cd97..61b5402 100644--- a/shared/prompt_enhancer/prompt_enhance_utils.py+++ b/shared/prompt_enhancer/prompt_enhance_utils.py@@ -164,6 +164,7 @@ def generate_cinematic_prompt(top_p: Optional[float] = None,top_k: Optional[int] = None,seed: Optional[int] = None,+ post_image_caption_hook = None,) -> List[str]:prompts = [prompt] if isinstance(prompt, str) else prompt@@ -200,6 +201,7 @@ def generate_cinematic_prompt(top_p,top_k,seed,+ post_image_caption_hook=post_image_caption_hook,)return prompts@@ -233,6 +235,17 @@ def _generate_t2v_prompt(for p in prompts]+ if hasattr(prompt_enhancer_model, "generate_messages"):+ return prompt_enhancer_model.generate_messages(+ messages,+ max_new_tokens,+ do_sample=do_sample,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ seed=seed,+ )+texts = [prompt_enhancer_tokenizer.apply_chat_template(m, tokenize=False, add_generation_prompt=True@@ -276,10 +289,20 @@ def _generate_i2v_prompt(top_p: Optional[float],top_k: Optional[int],seed: Optional[int],+ post_image_caption_hook = None,) -> List[str]:- image_captions = _generate_image_captions(- image_caption_model, image_caption_processor, first_frames- )+ if hasattr(image_caption_model, "generate_image_captions"):+ image_captions = image_caption_model.generate_image_captions(first_frames)+ else:+ image_captions = _generate_image_captions(+ image_caption_model, image_caption_processor, first_frames+ )+ if callable(post_image_caption_hook):+ if bool(getattr(prompt_enhancer_model, "_prompt_enhancer_use_vllm", False)):+ unload_runtime = getattr(prompt_enhancer_model, "unload", None)+ if callable(unload_runtime):+ unload_runtime()+ post_image_caption_hook()if len(image_captions) == 1 and len(image_captions) < len(prompts):image_captions *= len(prompts)messages = [@@ -290,6 +313,17 @@ def _generate_i2v_prompt(for p, c in zip(prompts, image_captions)]+ if hasattr(prompt_enhancer_model, "generate_messages"):+ return prompt_enhancer_model.generate_messages(+ messages,+ max_new_tokens,+ do_sample=do_sample,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ seed=seed,+ )+texts = [prompt_enhancer_tokenizer.apply_chat_template(m, tokenize=False, add_generation_prompt=Truediff --git a/shared/prompt_enhancer/qwen35_text.py b/shared/prompt_enhancer/qwen35_text.pynew file mode 100644index 0000000..d5b552f--- /dev/null+++ b/shared/prompt_enhancer/qwen35_text.py@@ -0,0 +1,644 @@+from __future__ import annotations++import gc+import importlib+import json+import os+import re+import types+from collections import OrderedDict++import torch+from mmgp import offload+from tqdm.auto import tqdm+from shared.utils import files_locator as fl+from shared.llm_engines.nanovllm import SamplingParams+from shared.llm_engines.nanovllm.models.qwen3_5 import Qwen3_5DynamicCache, Qwen3_5ForCausalLM, clear_qwen35_runtime_caches+from shared.llm_engines.nanovllm.utils.context import reset_context+from shared.llm_engines.nanovllm.vllm_support import (+ NanoVllmTextEngine,+ probe_vllm_runtime,+ resolve_lm_decoder_engine,+)+from shared.qtypes.gguf import materialize_module_source_tensors++from .qwen3_5 import register_qwen35_config+from .qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig+from .qwen35_vl import (+ enhancer_quantization_GGUF,+ enhancer_quantization_QUANTO_INT8,+ _collect_suppressed_token_ids,+ _collect_stop_token_ids,+ _load_qwen35_tokenizer,+ _resolve_qwen35_asset_file,+ _resolve_qwen35_assets_dir,+ get_qwen35_variant_spec,+ get_qwen35_text_gguf_path,+)+++QWEN35_TEXT_VLLM_SWITCH_ENV = "WGP_QWEN35_PROMPT_ENHANCER_VLLM"+QWEN35_TEXT_VLLM_CUDAGRAPH_ENV = "WGP_QWEN35_PROMPT_ENHANCER_VLLM_CUDAGRAPH"+QWEN35_GGUF_LLAMACPP_ENV = "WGP_GGUF_LLAMACPP_CUDA"+QWEN35_PROMPT_MIN_NEW_TOKENS = 4+++def _env_enabled(name: str, default: bool = True) -> bool:+ raw = str(os.environ.get(name, "1" if default else "0")).strip().lower()+ return raw in ("1", "true", "yes", "y", "on")+++def _resolve_gguf_model_path(model_path: str | None, assets_dir: str, variant: str | None = None) -> str:+ if model_path is not None:+ resolved = fl.locate_file(model_path, error_if_none=False) or model_path+ if not os.path.isfile(resolved):+ raise FileNotFoundError(f"Missing Qwen3.5 GGUF checkpoint: {resolved}")+ return resolved++ assets_dir = _resolve_qwen35_assets_dir(assets_dir, variant=variant)+ exact_path = get_qwen35_text_gguf_path(assets_dir, variant=variant)+ if os.path.isfile(exact_path):+ return exact_path+ raise FileNotFoundError(f"Missing expected Qwen3.5 GGUF checkpoint: {exact_path}")+++def _configure_qwen35_text_safe_legacy_kernels(enabled: bool) -> None:+ importlib.import_module("shared.llm_engines.nanovllm.models.qwen3_5").configure_qwen35_safe_legacy_kernels(bool(enabled))+++def get_qwen35_text_assets_dir(assets_dir: str, variant: str | None = None) -> str:+ return _resolve_qwen35_assets_dir(assets_dir, variant=variant)+++def get_qwen35_text_quanto_int8_path(assets_dir: str, variant: str | None = None) -> str:+ assets_dir = _resolve_qwen35_assets_dir(assets_dir, variant=variant)+ filename = get_qwen35_variant_spec(variant)["text_int8_filename"]+ return _resolve_qwen35_asset_file(assets_dir, filename, variant=variant, error_if_none=False) or os.path.join(assets_dir, filename)++def _load_text_config(config_path: str) -> Qwen3_5TextConfig:+ with open(config_path, "r", encoding="utf-8") as reader:+ config = json.load(reader)+ text_config = config.get("text_config", config)+ return Qwen3_5TextConfig(**text_config)++def _ensure_tied_output_weight(new_sd, tied_map):+ if "token_embd.weight" not in new_sd:+ return tied_map+ new_sd.pop("output.weight", None)+ tied_map = dict(tied_map or {})+ tied_list = list(tied_map.get("token_embd.weight", []))+ if "output.weight" not in tied_list:+ tied_list.append("output.weight")+ tied_map["token_embd.weight"] = tied_list+ return tied_map+++def _interleave_qwen35_ssm_heads(tensor: torch.Tensor) -> torch.Tensor:+ if tensor.ndim == 0 or tensor.shape[0] % 2 != 0:+ return tensor+ half = tensor.shape[0] // 2+ if tensor.ndim == 1:+ output = torch.empty_like(tensor)+ output[0::2] = tensor[:half]+ output[1::2] = tensor[half:]+ return output+ first_half, second_half = tensor[:half], tensor[half:]+ return torch.stack((first_half, second_half), dim=1).flatten(0, 1)+++def _build_qwen35_gguf_preprocess_sd(tie_output_to_embeddings: bool = False):+ def preprocess_sd(sd, quant_map=None, tied_map=None):+ new_sd = OrderedDict()+ for name, tensor in sd.items():+ if name.startswith("mtp.") or name.startswith("v."):+ continue+ if name.endswith(".ssm_dt.bias"):+ name = name[:-5]+ if name.endswith(".ssm_dt") or name.endswith(".ssm_a"):+ tensor = _interleave_qwen35_ssm_heads(tensor)+ if name.endswith(".ssm_conv1d.weight"):+ tensor = tensor.unsqueeze(1)+ new_sd[name] = tensor+ if tie_output_to_embeddings:+ tied_map = _ensure_tied_output_weight(new_sd, tied_map)+ return new_sd, quant_map, tied_map++ return preprocess_sd+++def _clean_generated_text(text: str) -> str:+ text = re.sub(r"<think>.*?</think>", "\n", text, flags=re.DOTALL | re.IGNORECASE)+ text = re.sub(r"<think>.*$", "", text, flags=re.DOTALL | re.IGNORECASE)+ text = text.replace("<|im_end|>", "").replace("<|im_start|>", "")+ text = text.replace("\r\n", "\n").replace("\r", "\n")+ text = re.sub(r"[ \t]+\n", "\n", text)+ text = re.sub(r"\n{3,}", "\n\n", text)+ lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.split("\n")]+ return "\n".join(line for line in lines if line).strip()+++def _apply_top_k_top_p(logits: torch.Tensor, top_k: int | None, top_p: float | None) -> torch.Tensor:+ logits = logits.clone()++ if top_k is not None and top_k > 0 and top_k < logits.shape[-1]:+ threshold = torch.topk(logits, int(top_k), dim=-1).values[..., -1, None]+ logits = logits.masked_fill(logits < threshold, float("-inf"))++ if top_p is not None and 0.0 < float(top_p) < 1.0:+ sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)+ sorted_probs = torch.softmax(sorted_logits, dim=-1)+ cumulative_probs = torch.cumsum(sorted_probs, dim=-1)+ sorted_mask = cumulative_probs > float(top_p)+ sorted_mask[..., 1:] = sorted_mask[..., :-1].clone()+ sorted_mask[..., 0] = False+ sorted_logits = sorted_logits.masked_fill(sorted_mask, float("-inf"))+ logits.scatter_(dim=-1, index=sorted_indices, src=sorted_logits)++ return logits+++def _sample_next_token(+ logits: torch.Tensor,+ do_sample: bool,+ temperature: float | None,+ top_p: float | None,+ top_k: int | None,+ generator: torch.Generator | None,+ suppress_token_ids: set[int] | None = None,+) -> torch.Tensor:+ if suppress_token_ids:+ blocked = [token_id for token_id in suppress_token_ids if 0 <= token_id < logits.shape[-1]]+ if blocked:+ logits = logits.clone()+ logits[..., blocked] = float("-inf")+ if not do_sample:+ return torch.argmax(logits, dim=-1, keepdim=True)++ if temperature is None or float(temperature) <= 0:+ return torch.argmax(logits, dim=-1, keepdim=True)++ logits = logits / float(temperature)+ logits = _apply_top_k_top_p(logits, top_k=top_k, top_p=top_p)+ probs = torch.softmax(logits, dim=-1)+ return torch.multinomial(probs, num_samples=1, generator=generator)+++def _last_step_logits(logits: torch.Tensor) -> torch.Tensor:+ if logits.ndim == 3:+ return logits[:, -1, :]+ return logits+++def _make_sampling_generator(device: torch.device, seed_value: int | None) -> torch.Generator | None:+ if seed_value is None:+ return None+ if device.type == "cuda":+ generator = torch.Generator(device="cuda")+ else:+ generator = torch.Generator()+ generator.manual_seed(int(seed_value))+ return generator+++def _build_chat_prompt(tokenizer, message):+ text = tokenizer.apply_chat_template(+ message,+ tokenize=False,+ add_generation_prompt=True,+ enable_thinking=False,+ )+ return text.rstrip() + "\n"+++def _generate_messages_legacy(+ self,+ messages,+ max_new_tokens,+ do_sample=True,+ temperature=None,+ top_p=None,+ top_k=None,+ seed=None,+):+ reset_context()+ outputs = []+ tokenizer = self._prompt_enhancer_tokenizer+ stop_token_ids = getattr(self, "_prompt_enhancer_stop_token_ids", None)+ if stop_token_ids is None:+ stop_token_ids = _collect_stop_token_ids(tokenizer, self.config)+ self._prompt_enhancer_stop_token_ids = stop_token_ids+ stop_token_ids = set(stop_token_ids)+ suppress_token_ids = set(getattr(self, "_prompt_enhancer_suppress_token_ids", []) or [])++ for idx, message in enumerate(tqdm(messages, total=len(messages), desc="Qwen3.5 prompt enhancement", dynamic_ncols=True, leave=False)):+ text = _build_chat_prompt(tokenizer, message)+ model_inputs = tokenizer(text, return_tensors="pt")+ input_ids = model_inputs["input_ids"].to(self.device)+ position_ids = torch.arange(input_ids.shape[1], device=self.device, dtype=torch.long).unsqueeze(0)+ past_key_values = Qwen3_5DynamicCache(self.config)+ generated_tokens = []+ with torch.inference_mode():+ hidden_states = self(input_ids, positions=position_ids, past_key_values=past_key_values)+ logits = _last_step_logits(self.compute_logits(hidden_states))+ generator = _make_sampling_generator(logits.device, int(seed) + idx if seed is not None else None)++ min_new_tokens = int(getattr(self, "_prompt_enhancer_min_new_tokens", 0) or 0)+ step_iter = tqdm(+ range(int(max_new_tokens)),+ total=int(max_new_tokens),+ desc="Qwen3.5 prompt enhancement tokens",+ unit="tok",+ dynamic_ncols=True,+ leave=False,+ )+ for step in step_iter:+ step_suppress = suppress_token_ids | stop_token_ids if step < min_new_tokens else suppress_token_ids+ next_token = _sample_next_token(+ logits,+ do_sample=bool(do_sample),+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ generator=generator,+ suppress_token_ids=step_suppress,+ )+ token_id = int(next_token.item())+ generated_tokens.append(token_id)+ if token_id in stop_token_ids:+ break+ next_position = torch.tensor([[input_ids.shape[1] + len(generated_tokens) - 1]], device=self.device, dtype=torch.long)+ hidden_states = self(next_token, positions=next_position, past_key_values=past_key_values)+ logits = _last_step_logits(self.compute_logits(hidden_states))++ outputs.append(+ _clean_generated_text(+ tokenizer.decode(+ generated_tokens,+ skip_special_tokens=False,+ clean_up_tokenization_spaces=False,+ )+ )+ )++ return outputs+++def _normalize_vllm_sampling(do_sample, temperature, top_p, top_k):+ if not do_sample:+ return 1.0, None, 1+ if temperature is None or float(temperature) <= 0:+ return 1.0, None, 1+ normalized_top_p = top_p if top_p is not None and 0.0 < float(top_p) < 1.0 else None+ normalized_top_k = int(top_k) if top_k is not None and int(top_k) > 0 else None+ return float(temperature), normalized_top_p, normalized_top_k+++def _format_vllm_probe_failure(probe_result) -> str:+ checks = probe_result.get("checks", {}) if isinstance(probe_result, dict) else {}+ reasons = []+ if isinstance(checks, dict):+ for check_name, check_data in checks.items():+ if not isinstance(check_data, dict) or bool(check_data.get("ok", False)):+ continue+ msg = str(check_data.get("message", "failed")).replace("\n", " ").strip()+ if len(msg) > 160:+ msg = msg[:160] + "..."+ reasons.append(f"{check_name}={msg}")+ return "; ".join(reasons) if len(reasons) > 0 else "runtime probe failed"+++def _resolve_prompt_enhancer_engine(backend: str, requested_lm_engine: str, runtime_model_path: str | None):+ if runtime_model_path is None:+ return "legacy", "vllm runtime path is not configured"+ if not _env_enabled(QWEN35_TEXT_VLLM_SWITCH_ENV, default=True):+ return "legacy", f"disabled by {QWEN35_TEXT_VLLM_SWITCH_ENV}"+ requested_lm_engine = str(requested_lm_engine or "").strip().lower()+ resolved_engine = resolve_lm_decoder_engine(requested_lm_engine, ["vllm"])+ if resolved_engine != "vllm":+ requested_label = requested_lm_engine or "auto"+ if requested_lm_engine in ("", "vllm"):+ return "legacy", f"lm_decoder_engine={requested_label}; {_format_vllm_probe_failure(probe_vllm_runtime())}"+ return "legacy", f"lm_decoder_engine={requested_label}"+ probe_result = probe_vllm_runtime()+ if bool(probe_result.get("supported", False)):+ if _env_enabled(QWEN35_TEXT_VLLM_CUDAGRAPH_ENV, default=True):+ return "vllm", "cuda graph"+ return "vllm", "eager"+ return "legacy", _format_vllm_probe_failure(probe_result)+++def _use_vllm_prompt_enhancer(model) -> bool:+ if not bool(getattr(model, "_prompt_enhancer_use_vllm", False)):+ return False+ if not _env_enabled(QWEN35_TEXT_VLLM_SWITCH_ENV, default=True):+ return False+ if not torch.cuda.is_available():+ return False+ probe_result = probe_vllm_runtime()+ return bool(probe_result.get("supported", False))+++def _get_or_create_vllm_engine(model, usage_mode: str | None = None):+ register_qwen35_config()+ engine = getattr(model, "_prompt_enhancer_vllm_engine", None)+ active_mode = getattr(model, "_prompt_enhancer_vllm_mode", None)+ if engine is not None and usage_mode is not None and active_mode not in (None, usage_mode):+ try:+ engine.close()+ except Exception:+ pass+ engine = None+ model._prompt_enhancer_vllm_engine = None+ if engine is not None:+ if usage_mode is not None:+ model._prompt_enhancer_vllm_mode = usage_mode+ return engine++ runtime_model_path = getattr(model, "_prompt_enhancer_vllm_model_path", None)+ tokenizer = getattr(model, "_prompt_enhancer_tokenizer", None)+ if not runtime_model_path:+ raise RuntimeError("Qwen3.5 prompt enhancer vLLM runtime path is not configured.")+ if tokenizer is None:+ raise RuntimeError("Qwen3.5 prompt enhancer tokenizer is not configured.")+ force_eager = bool(getattr(model, "_prompt_enhancer_vllm_force_eager", False))+ enable_cudagraph = (not force_eager) and _env_enabled(QWEN35_TEXT_VLLM_CUDAGRAPH_ENV, default=True)++ engine = NanoVllmTextEngine(+ model=model,+ model_path=runtime_model_path,+ tokenizer=tokenizer,+ enforce_eager=not enable_cudagraph,+ )+ model._prompt_enhancer_vllm_engine = engine+ model._prompt_enhancer_vllm_mode = usage_mode+ return engine+++def _reset_vllm_sequence_state(model):+ for module in model.modules():+ reset_sequence_state = getattr(module, "reset_sequence_state", None)+ if callable(reset_sequence_state):+ reset_sequence_state()+ continue+ if hasattr(module, "conv_state"):+ module.conv_state = None+ if hasattr(module, "recurrent_state"):+ module.recurrent_state = None+++def _generate_messages_vllm(+ self,+ messages,+ max_new_tokens,+ do_sample=True,+ temperature=None,+ top_p=None,+ top_k=None,+ seed=None,+):+ reset_context()+ tokenizer = self._prompt_enhancer_tokenizer+ if len(messages) == 0:+ return []++ engine = _get_or_create_vllm_engine(self, usage_mode="text")+ outputs = []+ for idx, message in enumerate(tqdm(messages, total=len(messages), desc="Qwen3.5 prompt enhancement (vllm)", dynamic_ncols=True, leave=False)):+ prompt = _build_chat_prompt(tokenizer, message)+ try:+ prompt_len = len(tokenizer.encode(prompt))+ except Exception:+ prompt_len = 0+ engine.reserve_runtime(prompt_len=prompt_len, max_tokens=int(max_new_tokens), cfg_scale=1.0)+ engine._ensure_llm()+ if engine._llm is None:+ raise RuntimeError("Qwen3.5 prompt enhancer vLLM runtime is not available.")+ # The shared Qwen text model may have been used by captioning right before prompt+ # enhancement. Rebuild the vLLM runtime so CUDA graphs are always captured against+ # clean sequence/KV state for the current MMGP-loaded weights.+ engine.release_runtime_allocations()+ _reset_vllm_sequence_state(self)+ engine._llm.model_runner.ensure_runtime_ready()+ try:+ engine._llm.reset()+ except Exception:+ pass++ temp, normalized_top_p, normalized_top_k = _normalize_vllm_sampling(+ do_sample=bool(do_sample),+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ )+ sample_seed = None if seed is None else int(seed) + idx+ sampling_params = SamplingParams(+ temperature=temp,+ max_tokens=int(max_new_tokens),+ cfg_scale=1.0,+ top_k=normalized_top_k,+ top_p=normalized_top_p,+ ignore_eos=False,+ seed=sample_seed,+ )++ try:+ batch_outputs = engine._llm.generate(+ prompts=[prompt],+ sampling_params=sampling_params,+ use_tqdm=True,+ unconditional_prompts=None,+ )+ engine._last_failure_reason = ""+ except Exception as exc:+ engine._last_failure_reason = str(exc)+ raise+ finally:+ reset_context()++ text, _ = engine._extract_text_and_tokens(batch_outputs[0] if batch_outputs else None)+ outputs.append(_clean_generated_text(text))+ return outputs+++def _generate_messages(+ self,+ messages,+ max_new_tokens,+ do_sample=True,+ temperature=None,+ top_p=None,+ top_k=None,+ seed=None,+):+ if _use_vllm_prompt_enhancer(self):+ return _generate_messages_vllm(+ self,+ messages,+ max_new_tokens,+ do_sample=do_sample,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ seed=seed,+ )+ return _generate_messages_legacy(+ self,+ messages,+ max_new_tokens,+ do_sample=do_sample,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ seed=seed,+ )+++def _unload_prompt_enhancer_text_runtime(self):+ engine = getattr(self, "_prompt_enhancer_vllm_engine", None)+ if engine is not None:+ try:+ engine.close()+ finally:+ self._prompt_enhancer_vllm_engine = None+ self._prompt_enhancer_vllm_mode = None+ try:+ clear_qwen35_runtime_caches()+ except Exception:+ pass+ reset_context()+ gc.collect()+ if torch.cuda.is_available():+ try:+ torch.cuda.synchronize()+ except Exception:+ pass+ try:+ torch.cuda.empty_cache()+ except Exception:+ pass+ try:+ torch.cuda.ipc_collect()+ except Exception:+ pass+++def _load_local_text_model(+ model_path: str,+ config_path: str,+ preprocess_sd=None,+ default_dtype: torch.dtype = torch.float16,+):+ config = _load_text_config(config_path)+ with torch.device("meta"):+ model = Qwen3_5ForCausalLM(config)++ offload.load_model_data(+ model,+ model_path,+ preprocess_sd=preprocess_sd,+ writable_tensors=False,+ default_dtype=default_dtype,+ )+ materialize_module_source_tensors(model)+ return model+++def _configure_qwen35_gguf_text_model(model, model_dtype: torch.dtype):+ for module in model.modules():+ if getattr(module, "layer_type", None) == "linear_attention" and hasattr(module, "_gguf_interleave_ssm_ab"):+ module._gguf_interleave_ssm_ab = True+ model.config.dtype = model_dtype+ model.config.torch_dtype = model_dtype++def load_qwen35_text_prompt_enhancer(+ model_path: str | None = None,+ assets_dir: str | None = None,+ default_dtype: torch.dtype = torch.float16,+ backend: str = enhancer_quantization_QUANTO_INT8,+ attn_implementation: str = "sdpa",+ requested_lm_engine: str = "",+ variant: str | None = None,+):+ del attn_implementation+ if assets_dir is None:+ raise ValueError("A local Qwen3.5 assets directory is required.")++ assets_dir = _resolve_qwen35_assets_dir(assets_dir, variant=variant)+ spec = get_qwen35_variant_spec(variant)+ text_assets_dir = get_qwen35_text_assets_dir(assets_dir, variant=variant)+ tokenizer_json = _resolve_qwen35_asset_file(assets_dir, "tokenizer.json", variant=variant, error_if_none=False) or os.path.join(assets_dir, "tokenizer.json")+ tokenizer_config = _resolve_qwen35_asset_file(assets_dir, "tokenizer_config.json", variant=variant, error_if_none=False) or os.path.join(assets_dir, "tokenizer_config.json")+ text_config_path = _resolve_qwen35_asset_file(text_assets_dir, "config.json", error_if_none=False) or os.path.join(text_assets_dir, "config.json")+ for required_file in (tokenizer_json, tokenizer_config, text_config_path):+ if not os.path.isfile(required_file):+ raise FileNotFoundError(f"Missing Qwen3.5 text prompt enhancer asset: {required_file}")++ tokenizer = _load_qwen35_tokenizer(assets_dir)+ chat_template_path = _resolve_qwen35_asset_file(text_assets_dir, "chat_template.jinja", error_if_none=False) or os.path.join(text_assets_dir, "chat_template.jinja")+ if os.path.isfile(chat_template_path):+ with open(chat_template_path, "r", encoding="utf-8") as reader:+ tokenizer.chat_template = reader.read()++ if backend == enhancer_quantization_GGUF:+ model_path = _resolve_gguf_model_path(model_path, assets_dir, variant=variant)+ preprocess_sd = _build_qwen35_gguf_preprocess_sd(+ tie_output_to_embeddings=bool(spec.get("tie_word_embeddings", False)),+ )+ runtime_model_path = text_assets_dir+ else:+ if model_path is None:+ model_path = get_qwen35_text_quanto_int8_path(assets_dir, variant=variant)+ preprocess_sd = None+ runtime_model_path = text_assets_dir++ engine_name, engine_detail = _resolve_prompt_enhancer_engine(+ backend=backend,+ requested_lm_engine=requested_lm_engine,+ runtime_model_path=runtime_model_path,+ )+ safe_legacy_mode = engine_name == "legacy"+ _configure_qwen35_text_safe_legacy_kernels(safe_legacy_mode)++ if not os.path.isfile(model_path):+ raise FileNotFoundError(f"Qwen3.5 text checkpoint not found: {model_path}")++ model = _load_local_text_model(+ model_path,+ text_config_path,+ preprocess_sd=preprocess_sd,+ default_dtype=default_dtype,+ )+ if backend == enhancer_quantization_GGUF:+ _configure_qwen35_gguf_text_model(model, default_dtype)++ model._prompt_enhancer_tokenizer = tokenizer+ model._prompt_enhancer_stop_token_ids = _collect_stop_token_ids(tokenizer, model.config)+ model._prompt_enhancer_suppress_token_ids = _collect_suppressed_token_ids(tokenizer)+ model._prompt_enhancer_min_new_tokens = (+ QWEN35_PROMPT_MIN_NEW_TOKENS+ if backend == enhancer_quantization_GGUF and _env_enabled(QWEN35_GGUF_LLAMACPP_ENV, default=True)+ else 0+ )+ model._prompt_enhancer_use_vllm = False+ model._prompt_enhancer_vllm_model_path = runtime_model_path+ model._prompt_enhancer_vllm_engine = None+ model._prompt_enhancer_vllm_mode = None+ model._prompt_enhancer_safe_legacy = safe_legacy_mode+ model._prompt_enhancer_use_vllm = engine_name == "vllm"+ model._prompt_enhancer_vllm_force_eager = engine_name == "vllm" and engine_detail != "cuda graph"+ print(f"[Prompt Enhancer][{spec['display_name']}] Text generation engine: {engine_name} ({engine_detail})")+ model.generate_messages = types.MethodType(_generate_messages, model)+ model.unload = types.MethodType(_unload_prompt_enhancer_text_runtime, model)+ model._offload_hooks = ["forward"]+ model.eval()+ return model+++load_qwen35_prompt_enhancer = load_qwen35_text_prompt_enhancer+++__all__ = [+ "get_qwen35_text_quanto_int8_path",+ "load_qwen35_prompt_enhancer",+ "load_qwen35_text_prompt_enhancer",+]diff --git a/shared/prompt_enhancer/qwen35_vl.py b/shared/prompt_enhancer/qwen35_vl.pynew file mode 100644index 0000000..3e57f02--- /dev/null+++ b/shared/prompt_enhancer/qwen35_vl.py@@ -0,0 +1,938 @@+from __future__ import annotations++import importlib+import json+import os+import re+import types+from collections import OrderedDict+from contextlib import nullcontext++import torch++from mmgp import offload+from transformers import AutoConfig, AutoTokenizer, Qwen2TokenizerFast, Qwen2VLImageProcessorFast, Qwen2VLProcessor+from transformers.modeling_outputs import BaseModelOutputWithPast+from transformers.models.qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor++from shared.llm_engines.nanovllm.models.qwen3_5 import Qwen3_5DynamicCache+from shared.llm_engines.nanovllm.utils.context import reset_context+from shared.qtypes.gguf import materialize_module_source_tensors+from shared.utils import files_locator as fl++from .qwen3_5 import load_qwen35_model_class+++UPSTREAM_MODELING_FILENAME = "modeling_qwen3_5.py"+enhancer_quantization_GGUF = "gguf"+enhancer_quantization_SAFETENSORS = "safetensors"+enhancer_quantization_QUANTO_INT8 = "quanto_int8"+QWEN35_GGUF_LLAMACPP_ENV = "WGP_GGUF_LLAMACPP_CUDA"+QWEN35_TEXT_GGUF_FILENAME = "Qwen3.5-9B-Abliterated-text-Q4_K_M.gguf"+QWEN35_VISION_FILENAME = "Qwen3.5-9B-vision_bf16.safetensors"+QWEN35_PROMPT_MIN_NEW_TOKENS = 4+QWEN35_ABLITERATED_REPO = "DeepBeepMeep/Wan2.1"+QWEN35_ABLITERATED_TEXT_REQUIRED_FILES = (+ "chat_template.jinja",+ "config.json",+)+QWEN35_4B_TEXT_GGUF_FILENAME = "Qwen3.5-4B-Abliterated-text-Q4_K_M.gguf"+QWEN35_4B_VISION_FILENAME = "Qwen3.5-4B-vision_bf16.safetensors"+QWEN35_4B_TEXT_INT8_FILENAME = "Qwen3.5-4B-Abliterated_quanto_bf16_int8.safetensors"+QWEN35_VARIANT_9B = "9b"+QWEN35_VARIANT_4B = "4b"+++def _configure_qwen35_vl_safe_legacy_kernels(enabled: bool) -> None:+ importlib.import_module("shared.prompt_enhancer.qwen3_5.modeling_qwen3_5").configure_qwen35_vl_safe_legacy_kernels(bool(enabled))++QWEN35_VARIANT_SPECS = {+ QWEN35_VARIANT_9B: {+ "display_name": "Qwen3.5-9B Abliterated",+ "assets_dir_name": "Qwen3_5_9B_Abliterated",+ "root_repo": QWEN35_ABLITERATED_REPO,+ "repo_subfolder": "Qwen3_5_9B_Abliterated",+ "root_files": [+ "chat_template.jinja",+ "config.json",+ "tokenizer.json",+ "tokenizer_config.json",+ "video_preprocessor_config.json",+ "vocab.json",+ ],+ "text_repo": QWEN35_ABLITERATED_REPO,+ "text_required_files": list(QWEN35_ABLITERATED_TEXT_REQUIRED_FILES),+ "text_int8_filename": "Qwen3.5-9B-abliterated_quanto_bf16_int8.safetensors",+ "gguf_repo": QWEN35_ABLITERATED_REPO,+ "text_gguf_filename": QWEN35_TEXT_GGUF_FILENAME,+ "vision_filename": QWEN35_VISION_FILENAME,+ "tie_word_embeddings": False,+ },+ QWEN35_VARIANT_4B: {+ "display_name": "Qwen3.5-4B Abliterated",+ "assets_dir_name": "Qwen3_5_4B_Abliterated",+ "root_repo": QWEN35_ABLITERATED_REPO,+ "repo_subfolder": "Qwen3_5_4B_Abliterated",+ "root_files": [+ "chat_template.jinja",+ "config.json",+ "generation_config.json",+ "merges.txt",+ "tokenizer.json",+ "tokenizer_config.json",+ "video_preprocessor_config.json",+ "vocab.json",+ ],+ "text_repo": None,+ "text_required_files": [],+ "text_int8_filename": QWEN35_4B_TEXT_INT8_FILENAME,+ "gguf_repo": QWEN35_ABLITERATED_REPO,+ "text_gguf_filename": QWEN35_4B_TEXT_GGUF_FILENAME,+ "vision_filename": QWEN35_4B_VISION_FILENAME,+ "tie_word_embeddings": True,+ },+}+QWEN35_VARIANT_ALIASES = {+ "": QWEN35_VARIANT_9B,+ "9": QWEN35_VARIANT_9B,+ "9b": QWEN35_VARIANT_9B,+ "9b_abliterated": QWEN35_VARIANT_9B,+ "qwen3.5-9b": QWEN35_VARIANT_9B,+ "qwen3.5-9b abliterated": QWEN35_VARIANT_9B,+ "4": QWEN35_VARIANT_4B,+ "4b": QWEN35_VARIANT_4B,+ "4b_abliterated": QWEN35_VARIANT_4B,+ "qwen3.5-4b": QWEN35_VARIANT_4B,+ "qwen3.5-4b abliterated": QWEN35_VARIANT_4B,+ "qwen3.5-4b_abliterated": QWEN35_VARIANT_4B,+}+++def get_qwen35_variant_spec(variant: str | None = None) -> dict:+ normalized = QWEN35_VARIANT_ALIASES.get(str(variant or "").strip().lower(), str(variant or "").strip().lower())+ if normalized not in QWEN35_VARIANT_SPECS:+ raise ValueError(f"Unsupported Qwen3.5 prompt enhancer variant: {variant}")+ return QWEN35_VARIANT_SPECS[normalized]+++def get_qwen35_assets_dir_name(variant: str | None = None) -> str:+ return get_qwen35_variant_spec(variant)["assets_dir_name"]+++def get_qwen35_prompt_enhancer_variant(model_no) -> str:+ return QWEN35_VARIANT_4B if int(model_no) == 3 else QWEN35_VARIANT_9B+++def _resolve_qwen35_assets_dir(assets_dir: str | None, variant: str | None = None, error_if_none: bool = True) -> str | None:+ folder_name = get_qwen35_assets_dir_name(variant)+ candidate = folder_name if assets_dir is None else assets_dir+ resolved = fl.locate_folder(candidate, error_if_none=False)+ if resolved is None and assets_dir is not None and candidate != folder_name:+ resolved = fl.locate_folder(folder_name, error_if_none=False)+ if resolved is None and error_if_none:+ raise FileNotFoundError(f"Missing Qwen3.5 assets folder '{candidate}'")+ return resolved+++def _resolve_qwen35_asset_file(+ assets_dir: str | None,+ filename: str,+ variant: str | None = None,+ error_if_none: bool = True,+) -> str | None:+ resolved_assets_dir = _resolve_qwen35_assets_dir(assets_dir, variant=variant, error_if_none=error_if_none)+ if resolved_assets_dir is None:+ return None+ path = os.path.join(resolved_assets_dir, filename)+ located = fl.locate_file(path, error_if_none=False)+ if located is None and error_if_none:+ raise FileNotFoundError(f"Missing Qwen3.5 asset: {path}")+ return located+++def get_qwen35_modeling_path() -> str:+ return os.path.join(os.path.dirname(__file__), "qwen3_5", UPSTREAM_MODELING_FILENAME)+++def ensure_qwen35_prompt_enhancer_assets(process_files_def, backend: str = enhancer_quantization_QUANTO_INT8, variant: str | None = None):+ spec = get_qwen35_variant_spec(variant)+ repo_subfolder = spec.get("repo_subfolder", "")+ qwen35_shared_files = list(spec["root_files"])+ qwen35_runtime_files = [spec["vision_filename"], spec["text_gguf_filename" if backend == enhancer_quantization_GGUF else "text_int8_filename"]]+ process_files_def(repoId=spec["root_repo"], sourceFolderList=[repo_subfolder], fileList=[qwen35_shared_files + qwen35_runtime_files])+ if spec.get("text_repo") and spec.get("text_required_files"):+ process_files_def(repoId=spec["text_repo"], sourceFolderList=[repo_subfolder], fileList=[list(spec["text_required_files"])])+ qwen35_modeling_path = get_qwen35_modeling_path()+ if not os.path.isfile(qwen35_modeling_path):+ raise FileNotFoundError(f"Missing repo-local Qwen3.5 modeling file: {qwen35_modeling_path}")+++def _load_qwen35_tokenizer(assets_dir: str):+ assets_dir = _resolve_qwen35_assets_dir(assets_dir)+ tokenizer_config_path = _resolve_qwen35_asset_file(assets_dir, "tokenizer_config.json")+ tokenizer_class = None+ tokenizer_config = {}+ if tokenizer_config_path is not None:+ try:+ with open(tokenizer_config_path, "r", encoding="utf-8") as reader:+ tokenizer_config = json.load(reader)+ tokenizer_class = tokenizer_config.get("tokenizer_class")+ except Exception:+ tokenizer_class = None+ chat_template = _load_qwen35_chat_template(assets_dir)+ if tokenizer_class == "TokenizersBackend":+ tokenizer_path = _resolve_qwen35_asset_file(assets_dir, "tokenizer.json", error_if_none=True)+ tokenizer = Qwen2TokenizerFast(+ tokenizer_file=tokenizer_path,+ bos_token=tokenizer_config.get("bos_token"),+ eos_token=tokenizer_config.get("eos_token"),+ unk_token=tokenizer_config.get("unk_token"),+ pad_token=tokenizer_config.get("pad_token"),+ clean_up_tokenization_spaces=bool(tokenizer_config.get("clean_up_tokenization_spaces", False)),+ split_special_tokens=bool(tokenizer_config.get("split_special_tokens", False)),+ model_max_length=int(tokenizer_config.get("model_max_length", 1000000000000000019884624838656)),+ )+ tokenizer.name_or_path = assets_dir+ for token_name, token_value in tokenizer_config.get("model_specific_special_tokens", {}).items():+ if token_value is not None:+ setattr(tokenizer, token_name, token_value)+ for token_name in ("image_token", "video_token", "vision_bos_token", "vision_eos_token", "audio_bos_token", "audio_eos_token", "audio_token"):+ token_value = tokenizer_config.get(token_name)+ if token_value is not None:+ setattr(tokenizer, token_name, token_value)+ if chat_template is not None:+ tokenizer.chat_template = chat_template+ return tokenizer+ tokenizer = AutoTokenizer.from_pretrained(assets_dir, trust_remote_code=True)+ if chat_template is not None:+ tokenizer.chat_template = chat_template+ return tokenizer+++def _load_qwen35_chat_template(assets_dir: str) -> str | None:+ assets_dir = _resolve_qwen35_assets_dir(assets_dir)+ chat_template_path = _resolve_qwen35_asset_file(assets_dir, "chat_template.jinja", error_if_none=False)+ if chat_template_path is None or not os.path.isfile(chat_template_path):+ return None+ with open(chat_template_path, "r", encoding="utf-8") as reader:+ return reader.read()+++def _load_qwen35_image_processor(assets_dir: str):+ assets_dir = _resolve_qwen35_assets_dir(assets_dir)+ preprocessor_config_path = _resolve_qwen35_asset_file(assets_dir, "preprocessor_config.json", error_if_none=False)+ if preprocessor_config_path is not None:+ return Qwen2VLImageProcessorFast.from_pretrained(assets_dir)++ video_preprocessor_config_path = _resolve_qwen35_asset_file(assets_dir, "video_preprocessor_config.json", error_if_none=True)+ with open(video_preprocessor_config_path, "r", encoding="utf-8") as reader:+ config = json.load(reader)+ config["image_processor_type"] = "Qwen2VLImageProcessorFast"+ config.pop("processor_class", None)+ config.pop("video_processor_type", None)+ return Qwen2VLImageProcessorFast(**config)+++def get_qwen35_text_gguf_path(assets_dir: str, variant: str | None = None) -> str:+ assets_dir = _resolve_qwen35_assets_dir(assets_dir, variant=variant)+ filename = get_qwen35_variant_spec(variant)["text_gguf_filename"]+ return _resolve_qwen35_asset_file(assets_dir, filename, variant=variant, error_if_none=False) or os.path.join(assets_dir, filename)+def _build_qwen35_vl_gguf_preprocess_sd(patch_shape):+ def preprocess_sd(sd, quant_map=None, tied_map=None):+ new_sd = OrderedDict()+ for name, tensor in sd.items():+ if name == "v.pos_embed.weight":+ target_name = "pos_embed.weight"+ target_tensor = tensor+ elif name == "v.patch_embed.bias":+ target_name = "patch_embed.proj.bias"+ target_tensor = tensor+ elif name == "v.patch_embed.weight":+ target_name = "patch_embed.proj.weight"+ target_tensor = tensor.reshape(*patch_shape)+ elif name.startswith("v.merger."):+ target_name = name[2:]+ target_tensor = tensor+ else:+ vision_match = re.match(r"^v\.blk\.(\d+)\.(.+)$", name)+ if not vision_match:+ continue+ layer_no = vision_match.group(1)+ suffix = vision_match.group(2)+ prefix = f"blocks.{layer_no}."+ target_name = None+ target_tensor = tensor+ if suffix == "attn_q.weight":+ target_name = prefix + "attn.q_proj.weight"+ elif suffix == "attn_q.bias":+ target_name = prefix + "attn.q_proj.bias"+ elif suffix == "attn_k.weight":+ target_name = prefix + "attn.k_proj.weight"+ elif suffix == "attn_k.bias":+ target_name = prefix + "attn.k_proj.bias"+ elif suffix == "attn_v.weight":+ target_name = prefix + "attn.v_proj.weight"+ elif suffix == "attn_v.bias":+ target_name = prefix + "attn.v_proj.bias"+ elif suffix == "attn_out.weight":+ target_name = prefix + "attn.proj.weight"+ elif suffix == "attn_out.bias":+ target_name = prefix + "attn.proj.bias"+ elif suffix.startswith("mlp.") or suffix.startswith("norm"):+ target_name = prefix + suffix+ if target_name is None:+ continue+ new_sd[target_name] = target_tensor+ return new_sd, quant_map, tied_map++ return preprocess_sd+++def _move_batch_to_device(batch, device):+ return {key: value.to(device) if torch.is_tensor(value) else value for key, value in batch.items()}+++def _resolve_execution_device(self, model_inputs=None) -> torch.device:+ if model_inputs is not None:+ for value in model_inputs.values():+ if torch.is_tensor(value):+ return value.device+ if torch.cuda.is_available():+ return torch.device("cuda", torch.cuda.current_device())+ try:+ return self.device+ except Exception:+ return torch.device("cpu")+++class _SharedQwen35TextAdapter(torch.nn.Module):+ def __init__(self, text_model: torch.nn.Module):+ super().__init__()+ object.__setattr__(self, "_shared_text_model", text_model)+ self.config = getattr(text_model, "config", None)++ @property+ def text_model(self):+ return object.__getattribute__(self, "_shared_text_model")++ @property+ def embed_tokens(self):+ return self.text_model.embed_tokens++ def get_input_embeddings(self):+ return self.text_model.embed_tokens++ def set_input_embeddings(self, value):+ self.text_model.token_embd = value++ def forward(+ self,+ input_ids=None,+ attention_mask=None,+ position_ids=None,+ past_key_values=None,+ inputs_embeds=None,+ use_cache=None,+ **kwargs,+ ):+ del kwargs+ if (input_ids is None) == (inputs_embeds is None):+ raise ValueError("Specify exactly one of input_ids or inputs_embeds for the shared Qwen3.5 text adapter.")+ if past_key_values is None and (use_cache is None or bool(use_cache)):+ past_key_values = Qwen3_5DynamicCache(self.config)+ hidden_states = self.text_model(+ input_ids=input_ids,+ inputs_embeds=inputs_embeds,+ position_ids=position_ids,+ attention_mask=attention_mask,+ past_key_values=past_key_values,+ use_cache=use_cache,+ )+ return BaseModelOutputWithPast(last_hidden_state=hidden_states, past_key_values=past_key_values)+++class _SharedQwen35LmHeadAdapter(torch.nn.Module):+ def __init__(self, lm_head: torch.nn.Module):+ super().__init__()+ object.__setattr__(self, "_shared_lm_head", lm_head)++ @property+ def lm_head(self):+ return object.__getattribute__(self, "_shared_lm_head")++ @property+ def weight(self):+ return self.lm_head.weight++ def forward(self, *args, **kwargs):+ return self.lm_head(*args, **kwargs)+++class _Qwen35CaptionWrapper(torch.nn.Module):+ def __init__(+ self,+ caption_runtime_model: torch.nn.Module,+ vision_tower_model: torch.nn.Module,+ text_model: torch.nn.Module,+ tokenizer,+ processor,+ ):+ super().__init__()+ object.__setattr__(self, "_caption_runtime_model", caption_runtime_model)+ object.__setattr__(self, "vision_tower_model", vision_tower_model)+ object.__setattr__(self, "model", caption_runtime_model.model)+ self.config = caption_runtime_model.config+ self.generation_config = caption_runtime_model.generation_config+ self._prompt_enhancer_tokenizer = tokenizer+ self._prompt_enhancer_processor = processor+ object.__setattr__(self, "_prompt_enhancer_text_model", text_model)+ self._prompt_enhancer_image_token_id = tokenizer.convert_tokens_to_ids("<|image_pad|>")+ self._prompt_enhancer_video_token_id = tokenizer.convert_tokens_to_ids("<|video_pad|>")++ def forward(self, *args, **kwargs):+ return self._caption_runtime_model(*args, **kwargs)++ def eval(self):+ self._caption_runtime_model.eval()+ self.vision_tower_model.eval()+ return super().eval()+++def _clean_generated_text(text: str) -> str:+ text = re.sub(r"<think>.*?</think>", "\n", text, flags=re.DOTALL | re.IGNORECASE)+ text = text.replace("<think>", "\n").replace("</think>", "\n")+ text = text.replace("<|im_end|>", "").replace("<|im_start|>", "")+ text = text.replace("\r\n", "\n").replace("\r", "\n")+ text = re.sub(r"[ \t]+\n", "\n", text)+ text = re.sub(r"\n{3,}", "\n\n", text)+ lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.split("\n")]+ return "\n".join(line for line in lines if line).strip()+++def _iter_token_ids(token_ids):+ if token_ids is None:+ return+ if isinstance(token_ids, (list, tuple, set)):+ for token_id in token_ids:+ yield from _iter_token_ids(token_id)+ return+ try:+ token_id = int(token_ids)+ except (TypeError, ValueError):+ return+ if token_id >= 0:+ yield token_id+++def _collect_stop_token_ids(tokenizer, *config_objects):+ stop_token_ids = []+ seen = set()++ def add(token_ids):+ for token_id in _iter_token_ids(token_ids):+ if token_id in seen:+ continue+ seen.add(token_id)+ stop_token_ids.append(token_id)++ add(getattr(tokenizer, "eos_token_id", None))+ for token in ("<|im_end|>", "<|endoftext|>"):+ add(tokenizer.convert_tokens_to_ids(token))++ for config_object in config_objects:+ if config_object is None:+ continue+ add(getattr(config_object, "eos_token_id", None))+ add(getattr(getattr(config_object, "text_config", None), "eos_token_id", None))++ return stop_token_ids+++def _collect_suppressed_token_ids(tokenizer):+ suppressed = []+ seen = set()+ for token in ("<think>", "</think>"):+ token_id = tokenizer.convert_tokens_to_ids(token)+ try:+ token_id = int(token_id)+ except (TypeError, ValueError):+ continue+ if token_id < 0 or token_id in seen:+ continue+ seen.add(token_id)+ suppressed.append(token_id)+ return suppressed+++def _resolve_stop_tokens_for_generation(self):+ stop_token_ids = getattr(self, "_prompt_enhancer_stop_token_ids", None)+ if stop_token_ids:+ return stop_token_ids[0] if len(stop_token_ids) == 1 else stop_token_ids+ return self.generation_config.eos_token_id+++def _get_qwen35_text_runtime_helpers():+ from . import qwen35_text as qwen35_text_mod++ return qwen35_text_mod+++def _apply_top_k_top_p(logits: torch.Tensor, top_k: int | None, top_p: float | None) -> torch.Tensor:+ logits = logits.clone()+ if top_k is not None and top_k > 0 and top_k < logits.shape[-1]:+ threshold = torch.topk(logits, int(top_k), dim=-1).values[..., -1, None]+ logits = logits.masked_fill(logits < threshold, float("-inf"))+ if top_p is not None and 0.0 < float(top_p) < 1.0:+ sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)+ sorted_probs = torch.softmax(sorted_logits, dim=-1)+ cumulative_probs = torch.cumsum(sorted_probs, dim=-1)+ sorted_mask = cumulative_probs > float(top_p)+ sorted_mask[..., 1:] = sorted_mask[..., :-1].clone()+ sorted_mask[..., 0] = False+ sorted_logits = sorted_logits.masked_fill(sorted_mask, float("-inf"))+ logits.scatter_(dim=-1, index=sorted_indices, src=sorted_logits)+ return logits+++def _sample_next_token(+ logits: torch.Tensor,+ do_sample: bool,+ temperature: float | None,+ top_p: float | None,+ top_k: int | None,+ suppress_token_ids: set[int] | None = None,+) -> torch.Tensor:+ if suppress_token_ids:+ blocked = [token_id for token_id in suppress_token_ids if 0 <= token_id < logits.shape[-1]]+ if blocked:+ logits = logits.clone()+ logits[..., blocked] = float("-inf")+ if not do_sample:+ return torch.argmax(logits, dim=-1, keepdim=True)+ if temperature is None or float(temperature) <= 0:+ return torch.argmax(logits, dim=-1, keepdim=True)+ logits = logits / float(temperature)+ logits = _apply_top_k_top_p(logits, top_k=top_k, top_p=top_p)+ probs = torch.softmax(logits, dim=-1)+ return torch.multinomial(probs, num_samples=1)+++def _trim_generated_ids(generated_ids: torch.Tensor, stop_token_ids: set[int]) -> list[list[int]]:+ rows = []+ for row in generated_ids.tolist():+ trimmed = []+ for token_id in row:+ if token_id in stop_token_ids:+ break+ trimmed.append(int(token_id))+ rows.append(trimmed)+ return rows+++def _generate_and_decode(+ self,+ model_inputs,+ max_new_tokens,+ do_sample,+ temperature,+ top_p,+ top_k,+ seed,+ progress_desc: str | None = None,+):+ device = _resolve_execution_device(self, model_inputs)+ if seed is None:+ rng_context = nullcontext()+ else:+ devices = []+ if isinstance(device, torch.device) and device.type == "cuda":+ devices = [device.index or 0]+ rng_context = torch.random.fork_rng(devices=devices) if devices else torch.random.fork_rng()++ with rng_context, torch.inference_mode():+ if seed is not None:+ torch.manual_seed(int(seed))+ if isinstance(device, torch.device) and device.type == "cuda":+ with torch.cuda.device(device):+ torch.cuda.manual_seed(int(seed))+ if hasattr(self, "model") and hasattr(self.model, "rope_deltas"):+ self.model.rope_deltas = None+ stop_token_ids = set(_iter_token_ids(_resolve_stop_tokens_for_generation(self)))+ suppress_token_ids = set(getattr(self, "_prompt_enhancer_suppress_token_ids", []) or [])+ current_inputs = {+ key: value+ for key, value in dict(model_inputs).items()+ if key not in {"use_cache", "return_dict", "output_attentions", "output_hidden_states"}+ }+ generated_steps = []+ min_new_tokens = int(getattr(self, "_prompt_enhancer_min_new_tokens", 0) or 0)+ step_iter = range(int(max_new_tokens))+ if progress_desc:+ step_iter = tqdm(+ step_iter,+ total=int(max_new_tokens),+ desc=progress_desc,+ unit="tok",+ dynamic_ncols=True,+ leave=False,+ )+ for step in step_iter:+ forward_inputs = {+ key: value+ for key, value in current_inputs.items()+ if value is not None+ }+ outputs = self(+ **forward_inputs,+ use_cache=True,+ return_dict=True,+ logits_to_keep=1,+ )+ logits = outputs.logits[:, -1, :]+ step_suppress = suppress_token_ids | stop_token_ids if step < min_new_tokens else suppress_token_ids+ next_token = _sample_next_token(+ logits,+ do_sample=bool(do_sample),+ temperature=temperature,+ top_p=top_p,+ top_k=top_k,+ suppress_token_ids=step_suppress,+ )+ generated_steps.append(next_token)+ if stop_token_ids and all(int(token_id) in stop_token_ids for token_id in next_token.view(-1).tolist()):+ break+ current_inputs = {+ "input_ids": next_token,+ "past_key_values": outputs.past_key_values,+ }+ if generated_steps:+ generated_ids = torch.cat(generated_steps, dim=1)+ else:+ generated_ids = model_inputs["input_ids"][:, :0]+ decoded_ids = _trim_generated_ids(generated_ids, stop_token_ids)+ return [+ _clean_generated_text(text)+ for text in self._prompt_enhancer_tokenizer.batch_decode(+ decoded_ids,+ skip_special_tokens=True,+ clean_up_tokenization_spaces=False,+ )+ ]+++def _prepare_multimodal_vllm_prompt(self, model_inputs):+ runtime_model = self._caption_runtime_model+ model_inputs = _move_batch_to_device(model_inputs, _resolve_execution_device(self, model_inputs))+ input_ids = model_inputs["input_ids"]+ attention_mask = model_inputs.get("attention_mask")+ image_grid_thw = model_inputs.get("image_grid_thw")+ video_grid_thw = model_inputs.get("video_grid_thw")+ mm_token_type_ids = model_inputs.get("mm_token_type_ids")+ with torch.inference_mode():+ if hasattr(runtime_model.model, "rope_deltas"):+ runtime_model.model.rope_deltas = None+ pixel_values = model_inputs.get("pixel_values")+ pixel_values_videos = model_inputs.get("pixel_values_videos")+ if pixel_values is not None:+ image_outputs = runtime_model.model.get_image_features(pixel_values, image_grid_thw, return_dict=True)+ if pixel_values_videos is not None:+ video_outputs = runtime_model.model.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True)+ inputs_embeds = runtime_model.model.get_input_embeddings()(input_ids)+ if pixel_values is not None:+ image_embeds = torch.cat(image_outputs.pooler_output, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)+ image_mask, _ = runtime_model.model.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds)+ inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)+ if pixel_values_videos is not None:+ video_embeds = torch.cat(video_outputs.pooler_output, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)+ _, video_mask = runtime_model.model.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds)+ inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)+ position_ids = runtime_model.model.compute_3d_position_ids(+ input_ids=input_ids,+ inputs_embeds=inputs_embeds,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ attention_mask=attention_mask,+ past_key_values=None,+ mm_token_type_ids=mm_token_type_ids,+ )+ active_mask = attention_mask[0].bool() if attention_mask is not None else torch.ones(input_ids.shape[1], device=input_ids.device, dtype=torch.bool)+ prompt_token_ids = [int(token_id) for token_id in input_ids[0][active_mask].tolist()]+ prompt_embeds = inputs_embeds[0, active_mask].contiguous()+ if position_ids is None:+ prompt_position_ids = None+ else:+ prompt_position_ids = position_ids[:, 0, active_mask].contiguous() if position_ids.ndim == 3 else position_ids[:, active_mask].contiguous()+ rope_deltas = getattr(runtime_model.model, "rope_deltas", None)+ position_offset = int(rope_deltas.reshape(-1)[0].item()) if torch.is_tensor(rope_deltas) and rope_deltas.numel() > 0 else 0+ return prompt_token_ids, prompt_embeds, prompt_position_ids, position_offset+++def _generate_image_captions_vllm(self, images):+ qwen35_text_mod = _get_qwen35_text_runtime_helpers()+ text_model = self._prompt_enhancer_text_model+ tokenizer = self._prompt_enhancer_tokenizer+ processor = self._prompt_enhancer_processor+ engine = qwen35_text_mod._get_or_create_vllm_engine(text_model, usage_mode="multimodal")+ outputs = []+ for image in images:+ message = [+ {+ "role": "user",+ "content": [+ {"type": "image", "image": image},+ {+ "type": "text",+ "text": "Describe this image accurately in one concise paragraph, focusing on the main subject, setting, and notable objects. Output only the description.",+ },+ ],+ }+ ]+ text = processor.apply_chat_template(+ message,+ tokenize=False,+ add_generation_prompt=True,+ enable_thinking=False,+ )+ model_inputs = processor(+ text=[text],+ images=[image],+ return_tensors="pt",+ padding=True,+ return_mm_token_type_ids=True,+ )+ prompt_token_ids, prompt_embeds, prompt_position_ids, position_offset = _prepare_multimodal_vllm_prompt(self, model_inputs)+ engine.reserve_runtime(prompt_len=len(prompt_token_ids), max_tokens=128, cfg_scale=1.0)+ engine._ensure_llm()+ if engine._llm is None:+ raise RuntimeError("Qwen3.5 caption vLLM runtime is not available.")+ engine.release_runtime_allocations()+ qwen35_text_mod._reset_vllm_sequence_state(text_model)+ engine._llm.model_runner.ensure_runtime_ready()+ try:+ engine._llm.reset()+ except Exception:+ pass+ temp, normalized_top_p, normalized_top_k = qwen35_text_mod._normalize_vllm_sampling(+ do_sample=False,+ temperature=None,+ top_p=None,+ top_k=None,+ )+ response = engine.generate_embedded(+ prompt_token_ids=prompt_token_ids,+ prompt_embeds=prompt_embeds,+ prompt_position_ids=prompt_position_ids,+ max_tokens=128,+ temperature=temp,+ top_p=normalized_top_p,+ top_k=normalized_top_k,+ cfg_scale=1.0,+ seed=None,+ use_tqdm=True,+ release_vram_after=True,+ ignore_eos=False,+ position_offset=position_offset,+ )+ outputs.append(_clean_generated_text("" if response is None else response.get("text", "")))+ reset_context()+ return outputs+++def _generate_image_captions(self, images):+ if _get_qwen35_text_runtime_helpers()._use_vllm_prompt_enhancer(self._prompt_enhancer_text_model):+ return _generate_image_captions_vllm(self, images)+ outputs = []+ processor = self._prompt_enhancer_processor+ for image in images:+ message = [+ {+ "role": "user",+ "content": [+ {"type": "image", "image": image},+ {+ "type": "text",+ "text": "Describe this image accurately in one concise paragraph, focusing on the main subject, setting, and notable objects. Output only the description.",+ },+ ],+ }+ ]+ text = processor.apply_chat_template(+ message,+ tokenize=False,+ add_generation_prompt=True,+ enable_thinking=False,+ )+ model_inputs = processor(+ text=[text],+ images=[image],+ return_tensors="pt",+ padding=True,+ return_mm_token_type_ids=True,+ )+ model_inputs = _move_batch_to_device(model_inputs, torch.device("cuda", torch.cuda.current_device()) if torch.cuda.is_available() else torch.device("cpu"))+ decoded = _generate_and_decode(+ self,+ model_inputs,+ max_new_tokens=128,+ do_sample=False,+ temperature=None,+ top_p=None,+ top_k=None,+ seed=None,+ progress_desc="Qwen3.5 image description tokens",+ )+ outputs.extend(decoded)+ return outputs+++def _unload_prompt_enhancer_vl_runtime(self):+ text_model = getattr(self, "_prompt_enhancer_text_model", None)+ unload = getattr(text_model, "unload", None)+ if callable(unload):+ unload()+ return None+++def get_qwen35_vision_path(assets_dir: str, variant: str | None = None) -> str:+ assets_dir = _resolve_qwen35_assets_dir(assets_dir, variant=variant)+ filename = get_qwen35_variant_spec(variant)["vision_filename"]+ return _resolve_qwen35_asset_file(assets_dir, filename, variant=variant, error_if_none=False) or os.path.join(assets_dir, filename)+++def load_qwen35_vl_prompt_enhancer(+ model_path: str | None = None,+ assets_dir: str | None = None,+ attn_implementation: str = "sdpa",+ text_model: torch.nn.Module | None = None,+ backend: str = enhancer_quantization_QUANTO_INT8,+ variant: str | None = None,+):+ assets_dir = _resolve_qwen35_assets_dir(assets_dir, variant=variant)+ if text_model is None:+ raise ValueError("A loaded Qwen3.5 text model is required to build the multimodal prompt enhancer.")+ legacy_safe_mode = bool(getattr(text_model, "_prompt_enhancer_safe_legacy", False))+ _configure_qwen35_vl_safe_legacy_kernels(legacy_safe_mode)+ if legacy_safe_mode:+ attn_implementation = "sdpa"++ config_path = _resolve_qwen35_asset_file(assets_dir, "config.json", error_if_none=True)+ modeling_path = get_qwen35_modeling_path()+ for required_file in (+ config_path,+ _resolve_qwen35_asset_file(assets_dir, "tokenizer.json", error_if_none=True),+ _resolve_qwen35_asset_file(assets_dir, "tokenizer_config.json", error_if_none=True),+ _resolve_qwen35_asset_file(assets_dir, "video_preprocessor_config.json", error_if_none=True),+ ):+ if not os.path.isfile(required_file):+ raise FileNotFoundError(f"Missing Qwen3.5VL asset: {required_file}")+ if not os.path.isfile(modeling_path):+ raise FileNotFoundError(f"Missing repo-local Qwen3.5 modeling file: {modeling_path}")++ if model_path is None:+ model_path = get_qwen35_vision_path(assets_dir, variant=variant)+ if not os.path.isfile(model_path):+ raise FileNotFoundError(f"Qwen3.5VL checkpoint not found: {model_path}")++ model_class = load_qwen35_model_class(modeling_path, class_name="Qwen3_5ForConditionalGeneration")+ config = AutoConfig.from_pretrained(assets_dir, trust_remote_code=True)+ config._attn_implementation = attn_implementation+ if hasattr(config, "text_config") and config.text_config is not None:+ config.text_config._attn_implementation = attn_implementation+ if hasattr(config, "vision_config") and config.vision_config is not None:+ config.vision_config._attn_implementation = attn_implementation+ with torch.device("meta"):+ model = model_class(config)+ model.model.visual = model.model.visual.__class__._from_config(config.vision_config)+ model.model.language_model = _SharedQwen35TextAdapter(text_model)+ model.lm_head = _SharedQwen35LmHeadAdapter(text_model.lm_head)+ if str(model_path).lower().endswith(".gguf"):+ preprocess_sd = _build_qwen35_vl_gguf_preprocess_sd(tuple(model.model.visual.patch_embed.proj.weight.shape))+ offload.load_model_data(+ model.model.visual,+ model_path,+ preprocess_sd=preprocess_sd,+ writable_tensors=False,+ default_dtype=torch.bfloat16,+ )+ materialize_module_source_tensors(model.model.visual)+ model.model.visual.to(dtype=torch.float16)+ else:+ offload.load_model_data(+ model.model.visual,+ model_path,+ modelPrefix="model.visual",+ writable_tensors=False,+ default_dtype=torch.bfloat16,+ )++ tokenizer = _load_qwen35_tokenizer(assets_dir)+ image_processor = _load_qwen35_image_processor(assets_dir)+ video_processor = Qwen2VLVideoProcessor.from_pretrained(assets_dir)+ processor = Qwen2VLProcessor(+ image_processor=image_processor,+ tokenizer=tokenizer,+ video_processor=video_processor,+ chat_template=tokenizer.chat_template,+ )+ processor.chat_template = tokenizer.chat_template++ caption_model = _Qwen35CaptionWrapper(+ caption_runtime_model=model,+ vision_tower_model=model.model.visual,+ text_model=text_model,+ tokenizer=tokenizer,+ processor=processor,+ )+ caption_model.generate_image_captions = types.MethodType(_generate_image_captions, caption_model)+ caption_model.unload = types.MethodType(_unload_prompt_enhancer_vl_runtime, caption_model)++ stop_token_ids = _collect_stop_token_ids(+ tokenizer,+ model.generation_config,+ model.config,+ )+ if not stop_token_ids:+ raise RuntimeError("Could not determine Qwen3.5VL stop token ids.")+ caption_model._prompt_enhancer_stop_token_ids = stop_token_ids+ caption_model._prompt_enhancer_suppress_token_ids = _collect_suppressed_token_ids(tokenizer)+ raw_fast_env = str(os.environ.get(QWEN35_GGUF_LLAMACPP_ENV, "1")).strip().lower()+ caption_model._prompt_enhancer_min_new_tokens = (+ QWEN35_PROMPT_MIN_NEW_TOKENS+ if backend == enhancer_quantization_GGUF and raw_fast_env in ("1", "true", "yes", "y", "on")+ else 0+ )+ caption_model.generation_config.eos_token_id = stop_token_ids[0] if len(stop_token_ids) == 1 else stop_token_ids++ pad_token_id = tokenizer.pad_token_id+ if pad_token_id is None:+ pad_token_id = stop_token_ids[0]+ caption_model.generation_config.pad_token_id = int(pad_token_id)++ if hasattr(caption_model.config, "text_config") and hasattr(caption_model.config.text_config, "eos_token_id"):+ caption_model.config.text_config.eos_token_id = stop_token_ids[0]+ if hasattr(caption_model.config, "text_config") and hasattr(caption_model.config.text_config, "pad_token_id"):+ caption_model.config.text_config.pad_token_id = int(pad_token_id)+ caption_model.eval()+ return caption_model, caption_model.vision_tower_model+++__all__ = [+ "QWEN35_ABLITERATED_TEXT_REQUIRED_FILES",+ "QWEN35_4B_TEXT_INT8_FILENAME",+ "QWEN35_VARIANT_9B",+ "QWEN35_VARIANT_4B",+ "enhancer_quantization_GGUF",+ "enhancer_quantization_SAFETENSORS",+ "enhancer_quantization_QUANTO_INT8",+ "QWEN35_TEXT_GGUF_FILENAME",+ "QWEN35_VISION_FILENAME",+ "UPSTREAM_MODELING_FILENAME",+ "get_qwen35_prompt_enhancer_variant",+ "get_qwen35_assets_dir_name",+ "get_qwen35_modeling_path",+ "get_qwen35_variant_spec",+ "ensure_qwen35_prompt_enhancer_assets",+ "get_qwen35_text_gguf_path",+ "get_qwen35_vision_path",+ "load_qwen35_vl_prompt_enhancer",+]diff --git a/shared/prompt_enhancer/qwen3_5/__init__.py b/shared/prompt_enhancer/qwen3_5/__init__.pynew file mode 100644index 0000000..0be60ab--- /dev/null+++ b/shared/prompt_enhancer/qwen3_5/__init__.py@@ -0,0 +1,45 @@+import logging++from transformers import AutoConfig+from transformers.models.auto.configuration_auto import CONFIG_MAPPING++from .configuration_qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig, Qwen3_5VisionConfig++logging.getLogger("fla.utils").setLevel(logging.ERROR)++from . import modeling_qwen3_5 as qwen35_modeling+++def register_qwen35_config():+ try:+ AutoConfig.register("qwen3_5", Qwen3_5Config)+ except Exception:+ pass+ try:+ AutoConfig.register("qwen3_5_text", Qwen3_5TextConfig)+ except Exception:+ pass+ try:+ CONFIG_MAPPING.register("qwen3_5", Qwen3_5Config)+ except Exception:+ pass+ try:+ CONFIG_MAPPING.register("qwen3_5_text", Qwen3_5TextConfig)+ except Exception:+ pass+++def load_qwen35_model_class(modeling_path: str | None = None, class_name: str = "Qwen3_5ForConditionalGeneration"):+ register_qwen35_config()+ try:+ return getattr(qwen35_modeling, class_name)+ except AttributeError as exc:+ raise AttributeError(f"Unsupported Qwen3.5 model class: {class_name}") from exc++__all__ = [+ "Qwen3_5Config",+ "Qwen3_5TextConfig",+ "Qwen3_5VisionConfig",+ "load_qwen35_model_class",+ "register_qwen35_config",+]diff --git a/shared/prompt_enhancer/qwen3_5/compat.py b/shared/prompt_enhancer/qwen3_5/compat.pynew file mode 100644index 0000000..ab1b4f8--- /dev/null+++ b/shared/prompt_enhancer/qwen3_5/compat.py@@ -0,0 +1,187 @@+from __future__ import annotations++import importlib+import torch+from contextlib import nullcontext+from transformers.masking_utils import create_causal_mask as hf_create_causal_mask+++class _InitCompat:+ @staticmethod+ def ones_(tensor):+ return torch.nn.init.ones_(tensor)++ @staticmethod+ def zeros_(tensor):+ return torch.nn.init.zeros_(tensor)++ @staticmethod+ def copy_(tensor, value):+ with torch.no_grad():+ tensor.copy_(value)+ return tensor+++init = _InitCompat()+++def use_kernelized_func(_func):+ def decorator(obj):+ return obj++ return decorator+++def torch_compilable_check(condition, message):+ if not bool(condition):+ raise ValueError(message)+++def is_flash_attention_requested(config):+ return getattr(config, "_attn_implementation", None) == "flash_attention_2"+++def maybe_autocast(device_type=None, enabled=True):+ if enabled and torch.cuda.is_available() and device_type == "cuda":+ return torch.autocast("cuda")+ return nullcontext()+++def merge_with_config_defaults(func=None, **_kwargs):+ if func is None:+ def decorator(inner):+ return inner++ return decorator+ return func+++def is_causal_conv1d_available():+ return False+ if not torch.cuda.is_available():+ return False+ try:+ causal_conv1d = importlib.import_module("causal_conv1d")+ except Exception:+ return False+ return all(hasattr(causal_conv1d, name) for name in ("causal_conv1d_fn", "causal_conv1d_update"))+++def is_flash_linear_attention_available():+ if not torch.cuda.is_available():+ return False+ try:+ fla_modules = importlib.import_module("fla.modules")+ fla_ops = importlib.import_module("fla.ops.gated_delta_rule")+ except Exception:+ return False+ return all(+ hasattr(fla_modules, name)+ for name in ("FusedRMSNormGated",)+ ) and all(+ hasattr(fla_ops, name)+ for name in ("chunk_gated_delta_rule", "fused_recurrent_gated_delta_rule")+ )+++def capture_outputs(func=None, **_kwargs):+ if func is None:+ def decorator(inner):+ return inner++ return decorator+ return func+++def auto_docstring(*args, **kwargs):+ if args and callable(args[0]) and len(args) == 1 and not kwargs:+ return args[0]++ def decorator(obj):+ return obj++ return decorator+++def can_return_tuple(func=None, **_kwargs):+ if func is None:+ def decorator(inner):+ return inner++ return decorator+ return func+++class _MaskCompatCache:+ def __init__(self, cache):+ self._cache = cache++ def __getattr__(self, name):+ return getattr(self._cache, name)++ def get_mask_sizes(self, query_length, layer_idx):+ if torch.is_tensor(query_length):+ if query_length.ndim == 0:+ query_length = 1+ else:+ query_length = int(query_length.shape[0])+ else:+ query_length = int(query_length)+ return self._cache.get_mask_sizes(query_length, layer_idx)+++def _normalize_cache_position(cache_position: torch.Tensor) -> torch.Tensor:+ if not torch.is_tensor(cache_position):+ return cache_position+ if cache_position.ndim == 0:+ return cache_position.reshape(1)+ if cache_position.ndim > 1:+ cache_position = cache_position.reshape(-1)+ return cache_position.to(dtype=torch.long)+++def create_causal_mask(*args, **kwargs):+ if "inputs_embeds" in kwargs and "input_embeds" not in kwargs:+ kwargs["input_embeds"] = kwargs.pop("inputs_embeds")+ if "cache_position" not in kwargs:+ input_embeds = kwargs.get("input_embeds")+ position_ids = kwargs.get("position_ids")+ past_key_values = kwargs.get("past_key_values")+ if position_ids is not None:+ if position_ids.ndim == 0:+ cache_position = position_ids.reshape(1)+ elif position_ids.ndim == 1:+ cache_position = position_ids+ else:+ cache_position = position_ids[0]+ elif input_embeds is not None:+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0+ cache_position = torch.arange(+ past_seen_tokens,+ past_seen_tokens + input_embeds.shape[1],+ device=input_embeds.device,+ )+ else:+ raise TypeError("create_causal_mask() requires either position_ids or input_embeds to infer cache_position")+ kwargs["cache_position"] = cache_position+ kwargs["cache_position"] = _normalize_cache_position(kwargs["cache_position"])+ past_key_values = kwargs.get("past_key_values")+ if past_key_values is not None and hasattr(past_key_values, "get_mask_sizes"):+ kwargs["past_key_values"] = _MaskCompatCache(past_key_values)+ return hf_create_causal_mask(*args, **kwargs)+++__all__ = [+ "init",+ "auto_docstring",+ "can_return_tuple",+ "capture_outputs",+ "create_causal_mask",+ "is_causal_conv1d_available",+ "is_flash_attention_requested",+ "is_flash_linear_attention_available",+ "maybe_autocast",+ "merge_with_config_defaults",+ "torch_compilable_check",+ "use_kernelized_func",+]diff --git a/shared/prompt_enhancer/qwen3_5/configuration_qwen3_5.py b/shared/prompt_enhancer/qwen3_5/configuration_qwen3_5.pynew file mode 100644index 0000000..45f2702--- /dev/null+++ b/shared/prompt_enhancer/qwen3_5/configuration_qwen3_5.py@@ -0,0 +1,169 @@+from __future__ import annotations++from transformers.configuration_utils import PretrainedConfig, layer_type_validation+++class Qwen3_5TextConfig(PretrainedConfig):+ model_type = "qwen3_5_text"+ keys_to_ignore_at_inference = ["past_key_values"]++ base_model_tp_plan = {+ "layers.*.self_attn.q_proj": "colwise",+ "layers.*.self_attn.k_proj": "colwise",+ "layers.*.self_attn.v_proj": "colwise",+ "layers.*.self_attn.o_proj": "rowwise",+ "layers.*.self_attn.q_norm": "replicated_with_grad_allreduce",+ "layers.*.self_attn.k_norm": "replicated_with_grad_allreduce",+ "layers.*.mlp.gate_proj": "colwise",+ "layers.*.mlp.up_proj": "colwise",+ "layers.*.mlp.down_proj": "rowwise",+ }+ base_model_pp_plan = {+ "embed_tokens": (["input_ids"], ["inputs_embeds"]),+ "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),+ "norm": (["hidden_states"], ["hidden_states"]),+ }+ base_config_key = "text_config"++ def __init__(+ self,+ vocab_size=248320,+ hidden_size=4096,+ intermediate_size=12288,+ num_hidden_layers=32,+ num_attention_heads=16,+ num_key_value_heads=4,+ hidden_act="silu",+ max_position_embeddings=32768,+ initializer_range=0.02,+ rms_norm_eps=1e-6,+ use_cache=True,+ tie_word_embeddings=False,+ rope_parameters=None,+ attention_bias=False,+ attention_dropout=0.0,+ head_dim=256,+ linear_conv_kernel_dim=4,+ linear_key_head_dim=128,+ linear_value_head_dim=128,+ linear_num_key_heads=16,+ linear_num_value_heads=32,+ layer_types=None,+ pad_token_id=None,+ bos_token_id=None,+ eos_token_id=None,+ **kwargs,+ ):+ kwargs["ignore_keys_at_rope_validation"] = ["mrope_section", "mrope_interleaved"]+ kwargs.setdefault("partial_rotary_factor", 0.25)+ super().__init__(+ pad_token_id=pad_token_id,+ bos_token_id=bos_token_id,+ eos_token_id=eos_token_id,+ tie_word_embeddings=tie_word_embeddings,+ **kwargs,+ )+ self.vocab_size = vocab_size+ self.max_position_embeddings = max_position_embeddings+ self.hidden_size = hidden_size+ self.intermediate_size = intermediate_size+ self.num_hidden_layers = num_hidden_layers+ self.num_attention_heads = num_attention_heads+ self.num_key_value_heads = num_key_value_heads+ self.hidden_act = hidden_act+ self.initializer_range = initializer_range+ self.rms_norm_eps = rms_norm_eps+ self.use_cache = use_cache+ self.attention_bias = attention_bias+ self.attention_dropout = attention_dropout+ self.head_dim = head_dim+ self.rope_parameters = rope_parameters+ self.layer_types = layer_types+ if self.layer_types is None:+ interval_pattern = kwargs.get("full_attention_interval", 4)+ self.layer_types = [+ "linear_attention" if bool((i + 1) % interval_pattern) else "full_attention"+ for i in range(self.num_hidden_layers)+ ]+ layer_type_validation(self.layer_types)++ self.linear_conv_kernel_dim = linear_conv_kernel_dim+ self.linear_key_head_dim = linear_key_head_dim+ self.linear_value_head_dim = linear_value_head_dim+ self.linear_num_key_heads = linear_num_key_heads+ self.linear_num_value_heads = linear_num_value_heads+++class Qwen3_5VisionConfig(PretrainedConfig):+ model_type = "qwen3_5"+ base_config_key = "vision_config"++ def __init__(+ self,+ depth=27,+ hidden_size=1152,+ hidden_act="gelu_pytorch_tanh",+ intermediate_size=4304,+ num_heads=16,+ in_channels=3,+ patch_size=16,+ spatial_merge_size=2,+ temporal_patch_size=2,+ out_hidden_size=3584,+ num_position_embeddings=2304,+ initializer_range=0.02,+ **kwargs,+ ):+ super().__init__(**kwargs)+ self.depth = depth+ self.hidden_size = hidden_size+ self.hidden_act = hidden_act+ self.intermediate_size = intermediate_size+ self.num_heads = num_heads+ self.in_channels = in_channels+ self.patch_size = patch_size+ self.spatial_merge_size = spatial_merge_size+ self.temporal_patch_size = temporal_patch_size+ self.out_hidden_size = out_hidden_size+ self.num_position_embeddings = num_position_embeddings+ self.initializer_range = initializer_range+++class Qwen3_5Config(PretrainedConfig):+ model_type = "qwen3_5"+ sub_configs = {"vision_config": Qwen3_5VisionConfig, "text_config": Qwen3_5TextConfig}+ keys_to_ignore_at_inference = ["past_key_values"]++ def __init__(+ self,+ text_config=None,+ vision_config=None,+ image_token_id=248056,+ video_token_id=248057,+ vision_start_token_id=248053,+ vision_end_token_id=248054,+ tie_word_embeddings=False,+ **kwargs,+ ):+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)+ if isinstance(vision_config, dict):+ self.vision_config = self.sub_configs["vision_config"](**vision_config)+ elif vision_config is None:+ self.vision_config = self.sub_configs["vision_config"]()+ else:+ self.vision_config = vision_config++ if isinstance(text_config, dict):+ self.text_config = self.sub_configs["text_config"](**text_config)+ elif text_config is None:+ self.text_config = self.sub_configs["text_config"]()+ else:+ self.text_config = text_config++ self.image_token_id = image_token_id+ self.video_token_id = video_token_id+ self.vision_start_token_id = vision_start_token_id+ self.vision_end_token_id = vision_end_token_id+++__all__ = ["Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5VisionConfig"]diff --git a/shared/prompt_enhancer/qwen3_5/modeling_qwen3_5.py b/shared/prompt_enhancer/qwen3_5/modeling_qwen3_5.pynew file mode 100644index 0000000..6e23173--- /dev/null+++ b/shared/prompt_enhancer/qwen3_5/modeling_qwen3_5.py@@ -0,0 +1,2348 @@+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨+# This file was automatically generated from src/transformers/models/qwen3_5/modular_qwen3_5.py.+# Do NOT edit this file manually as any edits will be overwritten by the generation of+# the file from the modular. If any change should be done, please apply the change to the+# modular_qwen3_5.py file directly. One of our CI enforces this.+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨+# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import itertools+import logging as py_logging+from collections.abc import Callable+from dataclasses import dataclass+from typing import Any, Optional++import torch+import torch.nn.functional as F+from torch import nn++from transformers.activations import ACT2FN+from transformers.cache_utils import Cache+from transformers.generation import GenerationMixin+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs+from transformers.modeling_layers import GenericForSequenceClassification, GradientCheckpointingLayer+from transformers.modeling_outputs import (+ BaseModelOutputWithPast,+ BaseModelOutputWithPooling,+ CausalLMOutputWithPast,+ ModelOutput,+)+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel+from transformers.processing_utils import Unpack+from transformers.utils import TransformersKwargs, logging++from .compat import (+ auto_docstring,+ can_return_tuple,+ capture_outputs,+ create_causal_mask,+ init,+ is_causal_conv1d_available,+ is_flash_attention_requested,+ is_flash_linear_attention_available,+ maybe_autocast,+ merge_with_config_defaults,+ torch_compilable_check,+ use_kernelized_func,+)+from .configuration_qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig, Qwen3_5VisionConfig++py_logging.getLogger("fla.utils").setLevel(py_logging.ERROR)+++if is_causal_conv1d_available():+ from causal_conv1d import causal_conv1d_fn, causal_conv1d_update+else:+ causal_conv1d_update, causal_conv1d_fn = None, None++if is_flash_linear_attention_available():+ from fla.modules import FusedRMSNormGated+ from fla.ops.gated_delta_rule import chunk_gated_delta_rule, fused_recurrent_gated_delta_rule+else:+ chunk_gated_delta_rule, fused_recurrent_gated_delta_rule = None, None+ FusedRMSNormGated = None++_DEFAULT_CHUNK_GATED_DELTA_RULE = chunk_gated_delta_rule+_DEFAULT_FUSED_RECURRENT_GATED_DELTA_RULE = fused_recurrent_gated_delta_rule+_DEFAULT_FUSED_RMSNORM_GATED = FusedRMSNormGated+++def configure_qwen35_vl_safe_legacy_kernels(enabled: bool) -> None:+ global chunk_gated_delta_rule, fused_recurrent_gated_delta_rule, FusedRMSNormGated+ if enabled:+ chunk_gated_delta_rule = None+ fused_recurrent_gated_delta_rule = None+ FusedRMSNormGated = None+ return+ chunk_gated_delta_rule = _DEFAULT_CHUNK_GATED_DELTA_RULE+ fused_recurrent_gated_delta_rule = _DEFAULT_FUSED_RECURRENT_GATED_DELTA_RULE+ FusedRMSNormGated = _DEFAULT_FUSED_RMSNORM_GATED++if not hasattr(ALL_ATTENTION_FUNCTIONS, "get_interface"):+ def _get_attention_interface(name, default):+ try:+ return ALL_ATTENTION_FUNCTIONS[name]+ except Exception:+ return default++ ALL_ATTENTION_FUNCTIONS.get_interface = _get_attention_interface++logger = logging.get_logger(__name__)+++def _normalize_torch_dtype(dtype_value):+ if not isinstance(dtype_value, str):+ return dtype_value+ normalized_name = dtype_value.strip().lower().replace("torch.", "")+ return getattr(torch, normalized_name, dtype_value)+++class Qwen3_5DynamicCache:+ """+ A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the linear attention+ cache (which has a constant shape regardless of seq_len).++ This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`+ and `ssm_states` for gated deltanet cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor+ For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,+ while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).+ For linear attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),+ while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,+ and `recurrent_states` represents the recurrent state and has a shape of `(batch_size, d_inner, d_state)`.+ """++ is_compileable = False++ def __init__(self, config: Qwen3_5Config):+ super().__init__()+ self.layer_types = config.layer_types+ self.transformer_layers = [+ i for i in range(config.num_hidden_layers) if self.layer_types[i] == "full_attention"+ ]+ self.last_linear_layer = len(self.layer_types) - 1 - self.layer_types[::-1].index("linear_attention")++ # Initialize everything to None -> will be lazy initialized to allow multi-gpu (device_map) inference+ self.conv_states = [None for _ in range(config.num_hidden_layers)]+ self.recurrent_states = [None for _ in range(config.num_hidden_layers)]+ self.key_cache = [None for _ in range(config.num_hidden_layers)]+ self.value_cache = [None for _ in range(config.num_hidden_layers)]++ def __len__(self):+ return len(self.layer_types)++ def update(+ self,+ key_states: torch.Tensor,+ value_states: torch.Tensor,+ layer_idx: int,+ cache_kwargs: dict[str, Any] | None = None,+ ) -> tuple[torch.Tensor, torch.Tensor]:+ if self.key_cache[layer_idx] is None:+ self.key_cache[layer_idx] = key_states+ self.value_cache[layer_idx] = value_states+ else:+ self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=2)+ self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=2)++ return self.key_cache[layer_idx], self.value_cache[layer_idx]++ def reorder_cache(self, beam_idx: torch.LongTensor):+ """Reorders the cache for beam search, given the selected beam indices."""+ for layer_idx in range(len(self.key_cache)):+ if self.key_cache[layer_idx] is not None:+ device = self.key_cache[layer_idx].device+ beam_idx = beam_idx.to(device)+ self.key_cache[layer_idx] = self.key_cache[layer_idx].index_select(0, beam_idx)+ self.value_cache[layer_idx] = self.value_cache[layer_idx].index_select(0, beam_idx)++ if self.conv_states[layer_idx] is not None:+ device = self.conv_states[layer_idx].device+ beam_idx = beam_idx.to(device)+ self.conv_states[layer_idx] = self.conv_states[layer_idx].index_select(0, beam_idx)+ self.recurrent_states[layer_idx] = self.recurrent_states[layer_idx].index_select(0, beam_idx)++ def get_seq_length(self, layer_idx: int | None = 0) -> int:+ """Returns the sequence length of the cached states. A layer index can be optionally passed."""+ # take any layer that contains cache and not empty tensor+ layer_idx = self.transformer_layers[0] if layer_idx not in self.transformer_layers else layer_idx+ if len(self.key_cache) <= layer_idx or self.key_cache[layer_idx] is None:+ return 0+ return self.key_cache[layer_idx].shape[-2]++ def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]:+ """+ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for+ the given layer at `layer_idx`.+ The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer.+ """+ kv_offset = 0+ past_seen_tokens = self.get_seq_length(layer_idx)+ kv_length = query_length + past_seen_tokens+ return kv_length, kv_offset++ @property+ def has_previous_state(self):+ """We have a previous state if the last linear (conv) layer was already updated."""+ return self.conv_states[self.last_linear_layer] is not None+++class Qwen3_5VisionRotaryEmbedding(nn.Module):+ inv_freq: torch.Tensor # fix linting for `register_buffer`++ def __init__(self, dim: int, theta: float = 10000.0) -> None:+ super().__init__()+ self.dim = dim+ self.theta = theta+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))+ self.register_buffer("inv_freq", inv_freq, persistent=False)++ def forward(self, seqlen: int) -> torch.Tensor:+ seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype)+ freqs = torch.outer(seq, self.inv_freq)+ return freqs+++class Qwen3_5TextRotaryEmbedding(nn.Module):+ inv_freq: torch.Tensor # fix linting for `register_buffer`++ def __init__(self, config: Qwen3_5TextConfig, device=None):+ super().__init__()+ self.max_seq_len_cached = config.max_position_embeddings+ self.original_max_seq_len = config.max_position_embeddings++ self.config = config++ self.rope_type = self.config.rope_parameters["rope_type"]+ rope_init_fn: Callable = self.compute_default_rope_parameters+ if self.rope_type != "default":+ rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]+ inv_freq, self.attention_scaling = rope_init_fn(self.config, device)++ self.register_buffer("inv_freq", inv_freq, persistent=False)+ self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)+ self.mrope_section = config.rope_parameters.get("mrope_section", [11, 11, 10])++ @staticmethod+ def compute_default_rope_parameters(+ config: Qwen3_5TextConfig | None = None,+ device: Optional["torch.device"] = None,+ seq_len: int | None = None,+ ) -> tuple["torch.Tensor", float]:+ """+ Computes the inverse frequencies according to the original RoPE implementation+ Args:+ config ([`~transformers.PreTrainedConfig`]):+ The model configuration.+ device (`torch.device`):+ The device to use for initialization of the inverse frequencies.+ seq_len (`int`, *optional*):+ The current sequence length. Unused for this type of RoPE.+ Returns:+ Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the+ post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).+ """+ base = config.rope_parameters["rope_theta"]+ partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)+ head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads+ dim = int(head_dim * partial_rotary_factor)++ attention_factor = 1.0 # Unused in this type of RoPE++ # Compute the inverse frequencies+ inv_freq = 1.0 / (+ base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)+ )+ return inv_freq, attention_factor++ @torch.no_grad()+ @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)+ def forward(self, x, position_ids):+ # In contrast to other models, Qwen3_5 has different position ids for the grids+ # So we expand the inv_freq to shape (3, ...)+ if position_ids.ndim == 2:+ position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)+ inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)+ position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions)++ device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"+ with maybe_autocast(device_type=device_type, enabled=False): # Force float32+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3)+ freqs = self.apply_interleaved_mrope(freqs, self.mrope_section)+ emb = torch.cat((freqs, freqs), dim=-1)+ cos = emb.cos() * self.attention_scaling+ sin = emb.sin() * self.attention_scaling++ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)++ def apply_interleaved_mrope(self, freqs, mrope_section):+ """Apply interleaved MRoPE to 3D rotary embeddings.+ Reorganizes frequency layout from chunked [TTT...HHH...WWW] to+ interleaved [THWTHWTHW...TT], preserving frequency continuity.+ args:+ x: (3, bs, seq_len, head_dim // 2)+ mrope_section: (3,)+ returns:+ x_t: (bs, seq_len, head_dim // 2)+ """+ freqs_t = freqs[0] # just overwrite the first dimension T+ for dim, offset in enumerate((1, 2), start=1): # H, W+ length = mrope_section[dim] * 3+ idx = slice(offset, length, 3)+ freqs_t[..., idx] = freqs[dim, ..., idx]+ return freqs_t+++class Qwen3_5RMSNormGated(nn.Module):+ def __init__(self, hidden_size, eps=1e-6, **kwargs):+ super().__init__()+ self.weight = nn.Parameter(torch.ones(hidden_size))+ self.variance_epsilon = eps++ def forward(self, hidden_states, gate=None):+ input_dtype = hidden_states.dtype+ hidden_states = hidden_states.to(torch.float32)+ variance = hidden_states.pow(2).mean(-1, keepdim=True)+ # Norm before gate+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)+ hidden_states = self.weight * hidden_states.to(input_dtype)+ hidden_states = hidden_states * F.silu(gate.to(torch.float32))++ return hidden_states.to(input_dtype)+++def apply_mask_to_padding_states(hidden_states, attention_mask):+ """+ Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66+ """+ # NOTE: attention mask is a 2D boolean tensor+ if attention_mask is not None and attention_mask.shape[1] > 1 and attention_mask.shape[0] > 1:+ dtype = hidden_states.dtype+ hidden_states = (hidden_states * attention_mask[:, :, None]).to(dtype)++ return hidden_states+++is_fast_path_available = all(+ (causal_conv1d_fn, causal_conv1d_update, chunk_gated_delta_rule, fused_recurrent_gated_delta_rule)+)+++def torch_causal_conv1d_update(+ hidden_states,+ conv_state,+ weight,+ bias=None,+ activation=None,+):+ _, hidden_size, seq_len = hidden_states.shape+ state_len = conv_state.shape[-1]++ hidden_states_new = torch.cat([conv_state, hidden_states], dim=-1).to(weight.dtype)+ conv_state.copy_(hidden_states_new[:, :, -state_len:])+ out = F.conv1d(hidden_states_new, weight.unsqueeze(1), bias, padding=0, groups=hidden_size)+ out = F.silu(out[:, :, -seq_len:])+ out = out.to(hidden_states.dtype)+ return out+++def l2norm(x: torch.FloatTensor, dim: int = -1, eps: float = 1e-6):+ """This function is intended to align with the l2norm implementation in the FLA library."""+ inv_norm = torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)+ return x * inv_norm+++def torch_chunk_gated_delta_rule(+ query,+ key,+ value,+ g,+ beta,+ chunk_size=64,+ initial_state=None,+ output_final_state=False,+ use_qk_l2norm_in_kernel=False,+):+ initial_dtype = query.dtype+ if use_qk_l2norm_in_kernel:+ query = l2norm(query, dim=-1, eps=1e-6)+ key = l2norm(key, dim=-1, eps=1e-6)+ query, key, value, beta, g = [+ x.transpose(1, 2).contiguous().to(torch.float32) for x in (query, key, value, beta, g)+ ]++ batch_size, num_heads, sequence_length, k_head_dim = key.shape+ v_head_dim = value.shape[-1]+ pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size+ query = F.pad(query, (0, 0, 0, pad_size))+ key = F.pad(key, (0, 0, 0, pad_size))+ value = F.pad(value, (0, 0, 0, pad_size))+ beta = F.pad(beta, (0, pad_size))+ g = F.pad(g, (0, pad_size))+ total_sequence_length = sequence_length + pad_size+ scale = 1 / (query.shape[-1] ** 0.5)+ query = query * scale++ v_beta = value * beta.unsqueeze(-1)+ k_beta = key * beta.unsqueeze(-1)+ # reshape to chunks+ query, key, value, k_beta, v_beta = [+ x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) for x in (query, key, value, k_beta, v_beta)+ ]+ g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)+ mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)++ # chunk decay+ g = g.cumsum(dim=-1)+ decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()+ attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)+ for i in range(1, chunk_size):+ row = attn[..., i, :i].clone()+ sub = attn[..., :i, :i].clone()+ attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)+ attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)+ value = attn @ v_beta+ k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))+ last_recurrent_state = (+ torch.zeros(batch_size, num_heads, k_head_dim, v_head_dim).to(value)+ if initial_state is None+ else initial_state.to(value)+ )+ core_attn_out = torch.zeros_like(value)+ mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=1)++ # for each chunk+ for i in range(0, total_sequence_length // chunk_size):+ q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]+ attn = (q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]).masked_fill_(mask, 0)+ v_prime = (k_cumdecay[:, :, i]) @ last_recurrent_state+ v_new = v_i - v_prime+ attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_recurrent_state+ core_attn_out[:, :, i] = attn_inter + attn @ v_new+ last_recurrent_state = (+ last_recurrent_state * g[:, :, i, -1, None, None].exp()+ + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new+ )++ if not output_final_state:+ last_recurrent_state = None+ core_attn_out = core_attn_out.reshape(core_attn_out.shape[0], core_attn_out.shape[1], -1, core_attn_out.shape[-1])+ core_attn_out = core_attn_out[:, :, :sequence_length]+ core_attn_out = core_attn_out.transpose(1, 2).contiguous().to(initial_dtype)+ return core_attn_out, last_recurrent_state+++def torch_recurrent_gated_delta_rule(+ query, key, value, g, beta, initial_state, output_final_state, use_qk_l2norm_in_kernel=False+):+ initial_dtype = query.dtype+ if use_qk_l2norm_in_kernel:+ query = l2norm(query, dim=-1, eps=1e-6)+ key = l2norm(key, dim=-1, eps=1e-6)+ query, key, value, beta, g = [+ x.transpose(1, 2).contiguous().to(torch.float32) for x in (query, key, value, beta, g)+ ]++ batch_size, num_heads, sequence_length, k_head_dim = key.shape+ v_head_dim = value.shape[-1]+ scale = 1 / (query.shape[-1] ** 0.5)+ query = query * scale++ core_attn_out = torch.zeros(batch_size, num_heads, sequence_length, v_head_dim).to(value)+ last_recurrent_state = (+ torch.zeros(batch_size, num_heads, k_head_dim, v_head_dim).to(value)+ if initial_state is None+ else initial_state.to(value)+ )++ for i in range(sequence_length):+ q_t = query[:, :, i]+ k_t = key[:, :, i]+ v_t = value[:, :, i]+ g_t = g[:, :, i].exp().unsqueeze(-1).unsqueeze(-1)+ beta_t = beta[:, :, i].unsqueeze(-1)++ last_recurrent_state = last_recurrent_state * g_t+ kv_mem = (last_recurrent_state * k_t.unsqueeze(-1)).sum(dim=-2)+ delta = (v_t - kv_mem) * beta_t+ last_recurrent_state = last_recurrent_state + k_t.unsqueeze(-1) * delta.unsqueeze(-2)+ core_attn_out[:, :, i] = (last_recurrent_state * q_t.unsqueeze(-1)).sum(dim=-2)++ if not output_final_state:+ last_recurrent_state = None+ core_attn_out = core_attn_out.transpose(1, 2).contiguous().to(initial_dtype)+ return core_attn_out, last_recurrent_state+++class Qwen3_5GatedDeltaNet(nn.Module):+ def __init__(self, config: Qwen3_5Config, layer_idx: int):+ super().__init__()+ self.hidden_size = config.hidden_size+ self.num_v_heads = config.linear_num_value_heads+ self.num_k_heads = config.linear_num_key_heads+ self.head_k_dim = config.linear_key_head_dim+ self.head_v_dim = config.linear_value_head_dim+ self.key_dim = self.head_k_dim * self.num_k_heads+ self.value_dim = self.head_v_dim * self.num_v_heads++ self.conv_kernel_size = config.linear_conv_kernel_dim+ self.layer_idx = layer_idx+ self.activation = config.hidden_act+ self.act = ACT2FN[config.hidden_act]+ self.layer_norm_epsilon = config.rms_norm_eps++ # QKV+ self.conv_dim = self.key_dim * 2 + self.value_dim+ self.conv1d = nn.Conv1d(+ in_channels=self.conv_dim,+ out_channels=self.conv_dim,+ bias=False,+ kernel_size=self.conv_kernel_size,+ groups=self.conv_dim,+ padding=self.conv_kernel_size - 1,+ )++ # time step projection (discretization)+ # instantiate once and copy inv_dt in init_weights of PretrainedModel+ self.dt_bias = nn.Parameter(torch.ones(self.num_v_heads))++ A = torch.empty(self.num_v_heads).uniform_(0, 16)+ self.A_log = nn.Parameter(torch.log(A))++ if FusedRMSNormGated is None:+ self.norm = Qwen3_5RMSNormGated(self.head_v_dim, eps=self.layer_norm_epsilon)+ else:+ fused_norm_kwargs = {+ "eps": self.layer_norm_epsilon,+ "activation": self.activation,+ "dtype": _normalize_torch_dtype(config.dtype)+ if config.dtype is not None+ else torch.get_default_dtype(),+ }+ if torch.cuda.is_available():+ fused_norm_kwargs["device"] = torch.device("cuda", torch.cuda.current_device())+ self.norm = FusedRMSNormGated(self.head_v_dim, **fused_norm_kwargs)++ self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)++ self.causal_conv1d_fn = causal_conv1d_fn+ self.causal_conv1d_update = causal_conv1d_update or torch_causal_conv1d_update+ self.chunk_gated_delta_rule = chunk_gated_delta_rule or torch_chunk_gated_delta_rule+ self.recurrent_gated_delta_rule = fused_recurrent_gated_delta_rule or torch_recurrent_gated_delta_rule++ # if not is_fast_path_available:+ # logger.warning_once(+ # "The fast path is not available because one of the required library is not installed. Falling back to "+ # "torch implementation. To install follow https://github.com/fla-org/flash-linear-attention#installation and"+ # " https://github.com/Dao-AILab/causal-conv1d"+ # )++ self.in_proj_qkv = nn.Linear(self.hidden_size, self.key_dim * 2 + self.value_dim, bias=False)+ self.in_proj_z = nn.Linear(self.hidden_size, self.value_dim, bias=False)+ self.in_proj_b = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)+ self.in_proj_a = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)++ def forward(+ self,+ hidden_states: torch.Tensor,+ cache_params: Qwen3_5DynamicCache | None = None,+ attention_mask: torch.Tensor | None = None,+ ):+ hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)++ # Set up dimensions for reshapes later+ batch_size, seq_len, _ = hidden_states.shape++ use_precomputed_states = cache_params is not None and cache_params.has_previous_state and seq_len == 1++ # getting projected states from cache if it exists+ if cache_params is not None:+ conv_state = cache_params.conv_states[self.layer_idx]+ recurrent_state = cache_params.recurrent_states[self.layer_idx]++ mixed_qkv = self.in_proj_qkv(hidden_states)+ mixed_qkv = mixed_qkv.transpose(1, 2)++ z = self.in_proj_z(hidden_states)+ z = z.reshape(batch_size, seq_len, -1, self.head_v_dim)++ b = self.in_proj_b(hidden_states)+ a = self.in_proj_a(hidden_states)++ if use_precomputed_states:+ # 2. Convolution sequence transformation+ # NOTE: the conv state is updated in `causal_conv1d_update`+ conv_kernel = self.conv1d.weight.reshape(+ self.conv1d.weight.shape[0],+ self.conv1d.weight.shape[-1],+ )+ mixed_qkv = self.causal_conv1d_update(+ mixed_qkv,+ conv_state,+ conv_kernel,+ self.conv1d.bias,+ self.activation,+ )+ if mixed_qkv.dtype != hidden_states.dtype:+ mixed_qkv = mixed_qkv.to(hidden_states.dtype)+ else:+ if cache_params is not None:+ conv_state = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0))+ cache_params.conv_states[self.layer_idx] = conv_state+ conv_dtype = self.conv1d.weight.dtype+ conv_input = mixed_qkv if mixed_qkv.dtype == conv_dtype else mixed_qkv.to(conv_dtype)+ if self.causal_conv1d_fn is not None:+ conv_kernel = self.conv1d.weight.reshape(+ self.conv1d.weight.shape[0],+ self.conv1d.weight.shape[-1],+ )+ mixed_qkv = self.causal_conv1d_fn(+ x=conv_input,+ weight=conv_kernel,+ bias=self.conv1d.bias,+ activation=self.activation,+ seq_idx=None,+ )+ else:+ mixed_qkv = F.silu(self.conv1d(conv_input)[:, :, :seq_len])+ if mixed_qkv.dtype != hidden_states.dtype:+ mixed_qkv = mixed_qkv.to(hidden_states.dtype)++ mixed_qkv = mixed_qkv.transpose(1, 2)+ query, key, value = torch.split(+ mixed_qkv,+ [+ self.key_dim,+ self.key_dim,+ self.value_dim,+ ],+ dim=-1,+ )++ query = query.reshape(batch_size, seq_len, -1, self.head_k_dim)+ key = key.reshape(batch_size, seq_len, -1, self.head_k_dim)+ value = value.reshape(batch_size, seq_len, -1, self.head_v_dim)++ beta = b.sigmoid()+ # If the model is loaded in fp16, without the .float() here, A might be -inf+ g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)+ if self.num_v_heads // self.num_k_heads > 1:+ query = query.repeat_interleave(self.num_v_heads // self.num_k_heads, dim=2)+ key = key.repeat_interleave(self.num_v_heads // self.num_k_heads, dim=2)++ if not use_precomputed_states:+ core_attn_out, last_recurrent_state = self.chunk_gated_delta_rule(+ query,+ key,+ value,+ g=g,+ beta=beta,+ initial_state=None,+ output_final_state=cache_params is not None,+ use_qk_l2norm_in_kernel=True,+ )++ else:+ core_attn_out, last_recurrent_state = self.recurrent_gated_delta_rule(+ query,+ key,+ value,+ g=g,+ beta=beta,+ initial_state=recurrent_state,+ output_final_state=cache_params is not None,+ use_qk_l2norm_in_kernel=True,+ )++ # Update cache+ if cache_params is not None:+ cache_params.recurrent_states[self.layer_idx] = last_recurrent_state++ # reshape input data into 2D tensor+ core_attn_out = core_attn_out.reshape(-1, self.head_v_dim)+ z = z.reshape(-1, self.head_v_dim)+ core_attn_out = self.norm(core_attn_out, z)+ core_attn_out = core_attn_out.reshape(batch_size, seq_len, -1)++ output = self.out_proj(core_attn_out)+ return output+++def rotate_half(x):+ """Rotates half the hidden dims of the input."""+ x1 = x[..., : x.shape[-1] // 2]+ x2 = x[..., x.shape[-1] // 2 :]+ return torch.cat((-x2, x1), dim=-1)+++# Adapted from transformers.models.glm.modular_glm.apply_rotary_pos_emb+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):+ """Applies Rotary Position Embedding to the query and key tensors.++ Removes the interleaving of cos and sin from GLM++ Args:+ q (`torch.Tensor`): The query tensor.+ k (`torch.Tensor`): The key tensor.+ cos (`torch.Tensor`): The cosine part of the rotary embedding.+ sin (`torch.Tensor`): The sine part of the rotary embedding.+ unsqueeze_dim (`int`, *optional*, defaults to 1):+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.+ Returns:+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.+ """+ cos = cos.unsqueeze(unsqueeze_dim)+ sin = sin.unsqueeze(unsqueeze_dim)++ # Keep half or full tensor for later concatenation+ rotary_dim = cos.shape[-1]+ q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]+ k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]++ # Apply rotary embeddings on the first half or full tensor+ q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin)+ k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin)++ # Concatenate back to full shape+ q_embed = torch.cat([q_embed, q_pass], dim=-1)+ k_embed = torch.cat([k_embed, k_pass], dim=-1)+ return q_embed, k_embed+++def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:+ """+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)+ """+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape+ if n_rep == 1:+ return hidden_states+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)+++def eager_attention_forward(+ module: nn.Module,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attention_mask: torch.Tensor | None,+ scaling: float,+ dropout: float = 0.0,+ **kwargs: Unpack[TransformersKwargs],+):+ key_states = repeat_kv(key, module.num_key_value_groups)+ value_states = repeat_kv(value, module.num_key_value_groups)++ attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling+ if attention_mask is not None:+ attn_weights = attn_weights + attention_mask++ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)+ attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)+ attn_output = torch.matmul(attn_weights, value_states)+ attn_output = attn_output.transpose(1, 2).contiguous()++ return attn_output, attn_weights+++@use_kernelized_func(apply_rotary_pos_emb)+class Qwen3_5Attention(nn.Module):+ """Multi-headed attention from 'Attention Is All You Need' paper"""++ def __init__(self, config: Qwen3_5Config, layer_idx: int):+ super().__init__()+ self.config = config+ self.layer_idx = layer_idx+ self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)+ self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads+ self.scaling = self.head_dim**-0.5+ self.attention_dropout = config.attention_dropout+ self.is_causal = True+ self.q_proj = nn.Linear(+ config.hidden_size, config.num_attention_heads * self.head_dim * 2, bias=config.attention_bias+ )+ self.k_proj = nn.Linear(+ config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias+ )+ self.v_proj = nn.Linear(+ config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias+ )+ self.o_proj = nn.Linear(+ config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias+ )+ self.q_norm = Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps) # unlike olmo, only on the head dim!+ self.k_norm = Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps) # thus post q_norm does not need reshape++ def forward(+ self,+ hidden_states: torch.Tensor,+ position_embeddings: tuple[torch.Tensor, torch.Tensor],+ attention_mask: torch.Tensor | None,+ past_key_values: Cache | None = None,+ **kwargs: Unpack[FlashAttentionKwargs],+ ) -> tuple[torch.Tensor, torch.Tensor | None]:+ input_shape = hidden_states.shape[:-1]+ hidden_shape = (*input_shape, -1, self.head_dim)++ query_states, gate = torch.chunk(+ self.q_proj(hidden_states).view(*input_shape, -1, self.head_dim * 2), 2, dim=-1+ )+ gate = gate.reshape(*input_shape, -1)++ query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)+ key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)+ value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)++ cos, sin = position_embeddings+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)++ if past_key_values is not None:+ key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)++ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(+ self.config._attn_implementation, eager_attention_forward+ )++ attn_output, attn_weights = attention_interface(+ self,+ query_states,+ key_states,+ value_states,+ attention_mask,+ dropout=0.0 if not self.training else self.attention_dropout,+ scaling=self.scaling,+ **kwargs,+ )++ attn_output = attn_output.reshape(*input_shape, -1).contiguous()+ attn_output = attn_output * torch.sigmoid(gate)++ attn_output = self.o_proj(attn_output)+ return attn_output, attn_weights+++class Qwen3_5MLP(nn.Module):+ def __init__(self, config: Qwen3_5Config, intermediate_size: int):+ super().__init__()+ self.config = config+ self.hidden_size = config.hidden_size+ self.intermediate_size = intermediate_size+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)+ self.act_fn = ACT2FN[config.hidden_act]++ def forward(self, x):+ down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))+ return down_proj+++class Qwen3_5RMSNorm(nn.Module):+ def __init__(self, dim: int, eps: float = 1e-6):+ super().__init__()+ self.eps = eps+ self.weight = nn.Parameter(torch.zeros(dim))++ def _norm(self, x):+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)++ def forward(self, x):+ output = self._norm(x.float())+ # Llama does x.to(float16) * w whilst Qwen3_5 is (x * w).to(float16)+ # See https://github.com/huggingface/transformers/pull/29402+ output = output * (1.0 + self.weight.float())+ return output.type_as(x)++ def extra_repr(self):+ return f"{tuple(self.weight.shape)}, eps={self.eps}"+++class Qwen3_5DecoderLayer(GradientCheckpointingLayer):+ def __init__(self, config: Qwen3_5TextConfig, layer_idx: int):+ super().__init__()+ self.hidden_size = config.hidden_size+ self.layer_type = config.layer_types[layer_idx]+ if self.layer_type == "linear_attention":+ self.linear_attn = Qwen3_5GatedDeltaNet(config, layer_idx)+ elif self.layer_type == "full_attention":+ self.self_attn = Qwen3_5Attention(config, layer_idx)+ self.mlp = Qwen3_5MLP(config, config.intermediate_size)+ self.input_layernorm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)+ self.post_attention_layernorm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)++ def forward(+ self,+ hidden_states: torch.Tensor,+ position_embeddings: tuple[torch.Tensor, torch.Tensor],+ attention_mask: torch.Tensor | None = None,+ position_ids: torch.LongTensor | None = None,+ past_key_values: Cache | None = None,+ **kwargs: Unpack[TransformersKwargs],+ ) -> torch.FloatTensor:+ residual = hidden_states++ hidden_states = self.input_layernorm(hidden_states)++ # Token Mixer+ if self.layer_type == "linear_attention":+ hidden_states = self.linear_attn(+ hidden_states=hidden_states,+ cache_params=past_key_values,+ attention_mask=attention_mask,+ )+ elif self.layer_type == "full_attention":+ # Self Attention+ hidden_states, _ = self.self_attn(+ hidden_states=hidden_states,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_values=past_key_values,+ position_embeddings=position_embeddings,+ **kwargs,+ )++ hidden_states = residual + hidden_states++ # Fully Connected+ residual = hidden_states+ hidden_states = self.post_attention_layernorm(hidden_states)+ hidden_states = self.mlp(hidden_states)+ hidden_states = residual + hidden_states++ return hidden_states+++class Qwen3_5PreTrainedModel(PreTrainedModel):+ config: Qwen3_5Config+ base_model_prefix = "model"+ supports_gradient_checkpointing = True+ _no_split_modules = ["Qwen3_5DecoderLayer", "Qwen3_5VisionBlock"]+ _skip_keys_device_placement = "past_key_values"+ _supports_flash_attn = True+ _supports_sdpa = True+ _keys_to_ignore_on_load_unexpected = [r"^mtp.*"]+ _can_record_outputs = {+ "hidden_states": Qwen3_5DecoderLayer,+ "attentions": Qwen3_5Attention,+ }+ _is_stateful = True++ @torch.no_grad()+ def _init_weights(self, module):+ super()._init_weights(module)+ if isinstance(module, Qwen3_5GatedDeltaNet):+ init.ones_(module.dt_bias)+ init.copy_(module.A_log, torch.empty_like(module.A_log).uniform_(0, 16).log_())+ # We initialize with 0s to be 1 centered as the RMSNorm here does (1 + weight)+ elif isinstance(module, Qwen3_5RMSNorm):+ init.zeros_(module.weight)+ elif isinstance(module, Qwen3_5VisionRotaryEmbedding):+ inv_freq = 1.0 / (module.theta ** (torch.arange(0, module.dim, 2, dtype=torch.float) / module.dim))+ init.copy_(module.inv_freq, inv_freq)+++class Qwen3_5VisionMLP(nn.Module):+ def __init__(self, config):+ super().__init__()+ self.hidden_size = config.hidden_size+ self.intermediate_size = config.intermediate_size+ self.linear_fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=True)+ self.linear_fc2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=True)+ self.act_fn = ACT2FN[config.hidden_act]++ def forward(self, hidden_state):+ return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state)))+++class Qwen3_5VisionPatchEmbed(nn.Module):+ def __init__(self, config) -> None:+ super().__init__()+ self.patch_size = config.patch_size+ self.temporal_patch_size = config.temporal_patch_size+ self.in_channels = config.in_channels+ self.embed_dim = config.hidden_size++ kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size]+ self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True)++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ target_dtype = self.proj.weight.dtype+ hidden_states = hidden_states.view(+ -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size+ )+ hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim)+ return hidden_states+++class Qwen3_5VisionPatchMerger(nn.Module):+ def __init__(self, config: Qwen3_5VisionConfig, use_postshuffle_norm=False) -> None:+ super().__init__()+ self.hidden_size = config.hidden_size * (config.spatial_merge_size**2)+ self.use_postshuffle_norm = use_postshuffle_norm+ self.norm = nn.LayerNorm(self.hidden_size if use_postshuffle_norm else config.hidden_size, eps=1e-6)+ self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size)+ self.act_fn = nn.GELU()+ self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ x = self.norm(x.view(-1, self.hidden_size) if self.use_postshuffle_norm else x).view(-1, self.hidden_size)+ x = self.linear_fc2(self.act_fn(self.linear_fc1(x)))+ return x+++def apply_rotary_pos_emb_vision(+ q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor+) -> tuple[torch.Tensor, torch.Tensor]:+ orig_q_dtype = q.dtype+ orig_k_dtype = k.dtype+ q, k = q.float(), k.float()+ cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()+ q_embed = (q * cos) + (rotate_half(q) * sin)+ k_embed = (k * cos) + (rotate_half(k) * sin)+ q_embed = q_embed.to(orig_q_dtype)+ k_embed = k_embed.to(orig_k_dtype)+ return q_embed, k_embed+++class Qwen3_5VisionAttention(nn.Module):+ def __init__(self, config: Qwen3_5VisionConfig) -> None:+ super().__init__()+ self.dim = config.hidden_size+ self.num_heads = config.num_heads+ self.head_dim = self.dim // self.num_heads+ self.num_key_value_groups = 1 # needed for eager attention+ self.q_proj = nn.Linear(self.dim, self.dim, bias=True)+ self.k_proj = nn.Linear(self.dim, self.dim, bias=True)+ self.v_proj = nn.Linear(self.dim, self.dim, bias=True)+ self.proj = nn.Linear(self.dim, self.dim)+ self.scaling = self.head_dim**-0.5+ self.config = config+ self.attention_dropout = 0.0+ self.is_causal = False++ def forward(+ self,+ hidden_states: torch.Tensor,+ cu_seqlens: torch.Tensor,+ rotary_pos_emb: torch.Tensor | None = None,+ position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,+ **kwargs,+ ) -> torch.Tensor:+ seq_length = hidden_states.shape[0]+ query_states = self.q_proj(hidden_states).reshape(seq_length, self.num_heads, -1)+ key_states = self.k_proj(hidden_states).reshape(seq_length, self.num_heads, -1)+ value_states = self.v_proj(hidden_states).reshape(seq_length, self.num_heads, -1)+ cos, sin = position_embeddings+ query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin)++ query_states = query_states.transpose(0, 1).unsqueeze(0)+ key_states = key_states.transpose(0, 1).unsqueeze(0)+ value_states = value_states.transpose(0, 1).unsqueeze(0)++ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(+ self.config._attn_implementation, eager_attention_forward+ )++ if is_flash_attention_requested(self.config):+ # Flash Attention: Use cu_seqlens for variable length attention+ max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()+ attn_output, _ = attention_interface(+ self,+ query_states,+ key_states,+ value_states,+ attention_mask=None,+ scaling=self.scaling,+ dropout=0.0 if not self.training else self.attention_dropout,+ cu_seq_lens_q=cu_seqlens,+ cu_seq_lens_k=cu_seqlens,+ max_length_q=max_seqlen,+ max_length_k=max_seqlen,+ is_causal=False,+ **kwargs,+ )+ else:+ # Other implementations: Process each chunk separately+ lengths = cu_seqlens[1:] - cu_seqlens[:-1]+ splits = [+ torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states)+ ]++ attn_outputs = [+ attention_interface(+ self,+ q,+ k,+ v,+ attention_mask=None,+ scaling=self.scaling,+ dropout=0.0 if not self.training else self.attention_dropout,+ is_causal=False,+ **kwargs,+ )[0]+ for q, k, v in zip(*splits)+ ]+ attn_output = torch.cat(attn_outputs, dim=1)++ attn_output = attn_output.reshape(seq_length, -1).contiguous()+ attn_output = self.proj(attn_output)+ return attn_output+++class Qwen3_5VisionBlock(GradientCheckpointingLayer):+ def __init__(self, config, attn_implementation: str = "sdpa") -> None:+ super().__init__()+ self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)+ self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)+ self.attn = Qwen3_5VisionAttention(config=config)+ self.mlp = Qwen3_5VisionMLP(config=config)++ def forward(+ self,+ hidden_states: torch.Tensor,+ cu_seqlens: torch.Tensor,+ rotary_pos_emb: torch.Tensor | None = None,+ position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,+ **kwargs,+ ) -> torch.Tensor:+ hidden_states = hidden_states + self.attn(+ self.norm1(hidden_states),+ cu_seqlens=cu_seqlens,+ rotary_pos_emb=rotary_pos_emb,+ position_embeddings=position_embeddings,+ **kwargs,+ )+ hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))+ return hidden_states+++class Qwen3_5VisionModel(Qwen3_5PreTrainedModel):+ config: Qwen3_5VisionConfig+ input_modalities = ("image", "video")+ _no_split_modules = ["Qwen3_5VisionBlock"]+ _can_record_outputs = {+ "hidden_states": Qwen3_5VisionBlock,+ "attentions": Qwen3_5VisionAttention,+ }++ def __init__(self, config, *inputs, **kwargs) -> None:+ super().__init__(config, *inputs, **kwargs)+ self.spatial_merge_size = config.spatial_merge_size+ self.patch_size = config.patch_size+ self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size++ self.patch_embed = Qwen3_5VisionPatchEmbed(+ config=config,+ )++ self.pos_embed = nn.Embedding(config.num_position_embeddings, config.hidden_size)+ self.num_grid_per_side = int(config.num_position_embeddings**0.5)++ head_dim = config.hidden_size // config.num_heads+ self.rotary_pos_emb = Qwen3_5VisionRotaryEmbedding(head_dim // 2)++ self.blocks = nn.ModuleList([Qwen3_5VisionBlock(config) for _ in range(config.depth)])+ self.merger = Qwen3_5VisionPatchMerger(+ config=config,+ use_postshuffle_norm=False,+ )++ self.gradient_checkpointing = False++ self.post_init()++ def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:+ merge_size = self.spatial_merge_size+ grid_thw_list = grid_thw.tolist()++ max_hw = max(max(h, w) for _, h, w in grid_thw_list)+ freq_table = self.rotary_pos_emb(max_hw) # (max_hw, dim // 2)+ device = freq_table.device++ total_tokens = sum(t * h * w for t, h, w in grid_thw_list)+ pos_ids = torch.empty((total_tokens, 2), dtype=torch.long, device=device)++ offset = 0+ for num_frames, height, width in grid_thw_list:+ merged_h, merged_w = height // merge_size, width // merge_size++ block_rows = torch.arange(merged_h, device=device) # block row indices+ block_cols = torch.arange(merged_w, device=device) # block col indices+ intra_row = torch.arange(merge_size, device=device) # intra-block row offsets+ intra_col = torch.arange(merge_size, device=device) # intra-block col offsets++ # Compute full-resolution positions+ row_idx = block_rows[:, None, None, None] * merge_size + intra_row[None, None, :, None]+ col_idx = block_cols[None, :, None, None] * merge_size + intra_col[None, None, None, :]++ row_idx = row_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1)+ col_idx = col_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1)++ coords = torch.stack((row_idx, col_idx), dim=-1)++ if num_frames > 1:+ coords = coords.repeat(num_frames, 1)++ num_tokens = coords.shape[0]+ pos_ids[offset : offset + num_tokens] = coords+ offset += num_tokens++ embeddings = freq_table[pos_ids] # lookup rotary embeddings+ embeddings = embeddings.flatten(1)+ return embeddings++ def fast_pos_embed_interpolate(self, grid_thw):+ grid_thw_list = grid_thw.tolist()+ grid_ts = [row[0] for row in grid_thw_list]+ grid_hs = [row[1] for row in grid_thw_list]+ grid_ws = [row[2] for row in grid_thw_list]+ device = self.pos_embed.weight.device++ idx_list = [[] for _ in range(4)]+ weight_list = [[] for _ in range(4)]++ for t, h, w in grid_thw_list:+ h_idxs = torch.linspace(0, self.num_grid_per_side - 1, h)+ w_idxs = torch.linspace(0, self.num_grid_per_side - 1, w)++ h_idxs_floor = h_idxs.int()+ w_idxs_floor = w_idxs.int()+ h_idxs_ceil = (h_idxs.int() + 1).clip(max=self.num_grid_per_side - 1)+ w_idxs_ceil = (w_idxs.int() + 1).clip(max=self.num_grid_per_side - 1)++ dh = h_idxs - h_idxs_floor+ dw = w_idxs - w_idxs_floor++ base_h = h_idxs_floor * self.num_grid_per_side+ base_h_ceil = h_idxs_ceil * self.num_grid_per_side++ indices = [+ (base_h[None].T + w_idxs_floor[None]).flatten(),+ (base_h[None].T + w_idxs_ceil[None]).flatten(),+ (base_h_ceil[None].T + w_idxs_floor[None]).flatten(),+ (base_h_ceil[None].T + w_idxs_ceil[None]).flatten(),+ ]++ weights = [+ ((1 - dh)[None].T * (1 - dw)[None]).flatten(),+ ((1 - dh)[None].T * dw[None]).flatten(),+ (dh[None].T * (1 - dw)[None]).flatten(),+ (dh[None].T * dw[None]).flatten(),+ ]++ for i in range(4):+ idx_list[i].extend(indices[i].tolist())+ weight_list[i].extend(weights[i].tolist())++ idx_tensor = torch.tensor(idx_list, dtype=torch.long, device=device)+ weight_tensor = torch.tensor(weight_list, dtype=self.pos_embed.weight.dtype, device=device)+ pos_embeds = self.pos_embed(idx_tensor).to(device) * weight_tensor[:, :, None]+ patch_pos_embeds = pos_embeds[0] + pos_embeds[1] + pos_embeds[2] + pos_embeds[3]++ patch_pos_embeds = patch_pos_embeds.split([h * w for h, w in zip(grid_hs, grid_ws)])++ patch_pos_embeds_permute = []+ merge_size = self.config.spatial_merge_size+ for pos_embed, t, h, w in zip(patch_pos_embeds, grid_ts, grid_hs, grid_ws):+ pos_embed = pos_embed.repeat(t, 1)+ pos_embed = (+ pos_embed.view(t, h // merge_size, merge_size, w // merge_size, merge_size, -1)+ .permute(0, 1, 3, 2, 4, 5)+ .flatten(0, 4)+ )+ patch_pos_embeds_permute.append(pos_embed)+ patch_pos_embeds = torch.cat(patch_pos_embeds_permute)+ return patch_pos_embeds++ @merge_with_config_defaults+ @capture_outputs+ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor:+ """+ Args:+ hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`):+ The final hidden states of the model.+ grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`):+ The temporal, height and width of feature shape of each image in LLM.++ Returns:+ `torch.Tensor`: hidden_states.+ """+ hidden_states = self.patch_embed(hidden_states)++ pos_embeds = self.fast_pos_embed_interpolate(grid_thw)+ hidden_states = hidden_states + pos_embeds++ rotary_pos_emb = self.rot_pos_emb(grid_thw)++ seq_len, _ = hidden_states.size()+ hidden_states = hidden_states.reshape(seq_len, -1)+ rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)+ emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)+ position_embeddings = (emb.cos(), emb.sin())++ cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum(+ dim=0,+ # Select dtype based on the following factors:+ # - FA2 requires that cu_seqlens_q must have dtype int32+ # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw+ # See https://github.com/huggingface/transformers/pull/34852 for more information+ dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,+ )+ cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)++ for blk in self.blocks:+ hidden_states = blk(+ hidden_states,+ cu_seqlens=cu_seqlens,+ position_embeddings=position_embeddings,+ **kwargs,+ )++ merged_hidden_states = self.merger(hidden_states)++ return BaseModelOutputWithPooling(+ last_hidden_state=hidden_states,+ pooler_output=merged_hidden_states,+ )+++@dataclass+@auto_docstring(+ custom_intro="""+ Base class for Llava outputs, with hidden states and attentions.+ """+)+class Qwen3_5ModelOutputWithPast(ModelOutput):+ r"""+ past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):+ It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).++ Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see+ `past_key_values` input) to speed up sequential decoding.+ rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):+ The rope index difference between sequence length and multimodal rope.+ """++ last_hidden_state: torch.FloatTensor | None = None+ past_key_values: Cache | None = None+ hidden_states: tuple[torch.FloatTensor] | None = None+ attentions: tuple[torch.FloatTensor] | None = None+ rope_deltas: torch.LongTensor | None = None+++class Qwen3_5TextModel(Qwen3_5PreTrainedModel):+ def __init__(self, config: Qwen3_5TextConfig):+ super().__init__(config)+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)+ self.layers = nn.ModuleList(+ [Qwen3_5DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]+ )+ self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)+ self.rotary_emb = Qwen3_5TextRotaryEmbedding(config=config)+ self.gradient_checkpointing = False+ # Initialize weights and apply final processing+ self.post_init()++ @merge_with_config_defaults+ @capture_outputs+ @auto_docstring+ def forward(+ self,+ input_ids: torch.LongTensor | None = None,+ attention_mask: torch.Tensor | None = None,+ position_ids: torch.LongTensor | None = None,+ past_key_values: Cache | None = None,+ inputs_embeds: torch.FloatTensor | None = None,+ use_cache: bool | None = None,+ **kwargs: Unpack[TransformersKwargs],+ ) -> BaseModelOutputWithPast:+ if (input_ids is None) ^ (inputs_embeds is not None):+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")++ if inputs_embeds is None:+ inputs_embeds = self.embed_tokens(input_ids)++ if use_cache and past_key_values is None:+ past_key_values = Qwen3_5DynamicCache(config=self.config)++ # the hard coded `4` is for text, temporal, height and width.+ if position_ids is None:+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0+ position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens+ position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1)+ elif position_ids.ndim == 2:+ position_ids = position_ids[None, ...].expand(4, position_ids.shape[0], -1)++ if position_ids.ndim == 3 and position_ids.shape[0] == 4:+ text_position_ids = position_ids[0]+ position_ids = position_ids[1:]+ else:+ text_position_ids = None++ causal_mask = create_causal_mask(+ config=self.config,+ inputs_embeds=inputs_embeds,+ attention_mask=attention_mask,+ past_key_values=past_key_values,+ position_ids=text_position_ids,+ )+ linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values)++ hidden_states = inputs_embeds+ position_embeddings = self.rotary_emb(hidden_states, position_ids)++ for layer_idx, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]):+ layer_mask = linear_attn_mask if decoder_layer.layer_type == "linear_attention" else causal_mask++ hidden_states = decoder_layer(+ hidden_states,+ position_embeddings=position_embeddings,+ attention_mask=layer_mask,+ position_ids=text_position_ids,+ past_key_values=past_key_values,+ use_cache=use_cache,+ **kwargs,+ )++ hidden_states = self.norm(hidden_states)++ return Qwen3_5ModelOutputWithPast(+ last_hidden_state=hidden_states,+ past_key_values=past_key_values,+ )++ def _update_linear_attn_mask(self, attention_mask, past_key_values):+ """+ NOTE: Left-padding is used for linear attention mask.+ No need for zeroing states when+ 1. Cached forward+ 2. Attending to all inputs+ """+ linear_attn_mask = attention_mask+ if (past_key_values is not None and past_key_values.get_seq_length() > 0) or (+ attention_mask is not None and torch.all(attention_mask == 1)+ ):+ linear_attn_mask = None+ return linear_attn_mask+++@auto_docstring+class Qwen3_5Model(Qwen3_5PreTrainedModel):+ base_model_prefix = "model"+ _checkpoint_conversion_mapping = {}+ # Reference: fix gemma3 grad acc #37208+ accepts_loss_kwargs = False+ config: Qwen3_5Config+ _no_split_modules = ["Qwen3_5DecoderLayer", "Qwen3_5VisionBlock"]++ def __init__(self, config):+ super().__init__(config)+ self.visual = Qwen3_5VisionModel._from_config(config.vision_config)+ self.language_model = Qwen3_5TextModel._from_config(config.text_config)+ self.rope_deltas = None # cache rope_deltas here++ # Initialize weights and apply final processing+ self.post_init()++ def get_input_embeddings(self):+ return self.language_model.get_input_embeddings()++ def set_input_embeddings(self, value):+ self.language_model.set_input_embeddings(value)++ def get_vision_position_ids(+ self,+ start_position: int,+ grid_thw: list[int, int, int] | torch.Tensor,+ temp_merge_size: int = 1,+ spatial_merge_size: int = 1,+ time_interval: int = 1,+ device: str | torch.device | None = None,+ ):+ """+ Compute 3D positional indices for vision tokens derived from a single image or video input.++ The positions are generated from the input grid defined by temporal (T), height (H), and+ width (W) dimensions. Temporal and spatial dimensions can be downscaled according to the+ merge sizes used in the vision backbone. The resulting positions are offset by `start_position`.++ Args:+ start_position (`int`):+ Offset added to all computed positional indices.+ grid_thw (`Sequence[int]` or `torch.Tensor` of shape `(3,)`):+ The (T, H, W) grid representing the feature layout of the current image or video after patch embedding.+ temp_merge_size (`int`, *optional*):+ Factor by which the temporal dimension is reduced in the backbone. The temporal grid size is divided+ by this value. Defaults to 1.+ spatial_merge_size (`int`, *optional*):+ Factor by which the spatial dimensions (H and W) are reduced in the backbone. Both H and W are divided+ by this value. Defaults to 1.+ time_interval (`int`, *optional*):+ Spacing factor applied between consecutive temporal position indices.Defaults to 1.+ device (`str` or `torch.device`, *optional*):+ Device on which the resulting tensor is allocated. If `None`, uses the current default device.++ Returns:+ torch.LongTensor of shape (3, sequence_length):+ Positional indices for temporal, height, and width dimensions,+ flattened into sequence form and offset by `start_position`.+ """+ llm_grid_t, llm_grid_h, llm_grid_w = (+ grid_thw[0].item() // temp_merge_size,+ grid_thw[1].item() // spatial_merge_size,+ grid_thw[2].item() // spatial_merge_size,+ )++ image_seq_length = llm_grid_h * llm_grid_w * llm_grid_t+ position_width = torch.arange(start_position, start_position + llm_grid_w, device=device).repeat(+ llm_grid_h * llm_grid_t+ )+ position_height = torch.arange(start_position, start_position + llm_grid_h, device=device).repeat_interleave(+ llm_grid_w * llm_grid_t+ )+ position_temporal = torch.full((image_seq_length,), start_position, device=device, dtype=torch.long)+ position_temporal = position_temporal * time_interval+ vision_position_ids = torch.stack([position_temporal, position_height, position_width], dim=0)++ return vision_position_ids++ def get_rope_index(+ self,+ input_ids: torch.LongTensor,+ mm_token_type_ids: torch.IntTensor,+ image_grid_thw: torch.LongTensor | None = None,+ video_grid_thw: torch.LongTensor | None = None,+ attention_mask: torch.Tensor | None = None,+ **kwargs,+ ) -> tuple[torch.Tensor, torch.Tensor]:+ """+ Calculate the 3D rope index based on image and video's sizes. The utility expects a `vision + text`+ sequence and will error out otherwise. For pure text sequence, please rely on model's auto-inferred+ position ids. In a mixed vision + text sequence, vision tokens use 3D RoPE (temporal, height, width)+ while text tokens use standard 1D RoPE.++ Example:+ Temporal patches: 3; Height patches: 2; Width patches: 2+ Each vision input results in (temporal x height × width) positions. Here: 3 x 2 × 2 = 12 positions total.++ Temporal position IDs are spaced by:+ `interval = tokens_per_second * temporal_patch_size / fps`++ If fps = 1; tokens_per_second = 25; temporal_patch_size = 2, temporal IDs increase by 50 for each temporal patch:+ `[0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100]`++ Height IDs repeat per row: `[0, 0, 1, 1, ...]`+ Width IDs alternate per column: `[0, 1, 0, 1, ...]`+ Text tokens follow standard 1D RoPE and the position IDs grow consequently with a step of `1`++ Args:+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide+ it.+ mm_token_type_ids (`torch.IntTensor` of shape `(batch_size, sequence_length)`):+ Token type ids matching each modality to a different value in the input sequence, i.e. text (0), image (1), video (2).+ image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):+ The temporal, height and width of feature shape of each image in LLM.+ video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):+ The temporal, height and width of feature shape of each video in LLM.+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:++ - 1 for tokens that are **not masked**,+ - 0 for tokens that are **masked**.++ Returns:+ position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`)+ mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`)+ """+ spatial_merge_size = self.config.vision_config.spatial_merge_size++ mrope_position_deltas = []+ position_ids = torch.zeros(+ 3,+ input_ids.shape[0],+ input_ids.shape[1],+ dtype=input_ids.dtype,+ device=input_ids.device,+ )+ grid_iters = {+ 1: iter(image_grid_thw) if image_grid_thw is not None else None,+ 2: iter(video_grid_thw) if video_grid_thw is not None else None,+ }++ for batch_idx, current_input_ids in enumerate(input_ids):+ input_token_type = mm_token_type_ids[batch_idx]+ if attention_mask is not None:+ current_input_ids = current_input_ids[attention_mask[batch_idx].bool()]+ input_token_type = input_token_type[attention_mask[batch_idx].bool()]++ input_type_group = []+ for key, group in itertools.groupby(enumerate(input_token_type.tolist()), lambda x: x[1]):+ group = list(group)+ start_index = group[0][0]+ end_index = group[-1][0] + 1+ input_type_group.append((key, start_index, end_index))++ current_pos = 0+ llm_pos_ids_list = []+ for modality_type, start_idx, end_idx in input_type_group:+ # text == 0+ if modality_type == 0:+ text_len = end_idx - start_idx+ llm_pos_ids_list.append(+ torch.arange(text_len, device=input_ids.device).view(1, -1).expand(3, -1) + current_pos+ )+ current_pos += text_len+ # image == 1, video == 2+ else:+ grid_thw = next(grid_iters[modality_type])+ vision_position_ids = self.get_vision_position_ids(+ current_pos, grid_thw, 1, spatial_merge_size, device=input_ids.device+ )+ llm_pos_ids_list.append(vision_position_ids)+ current_pos += max(grid_thw[1], grid_thw[2]) // spatial_merge_size+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)+ if attention_mask is not None:+ position_ids[:, batch_idx, attention_mask[batch_idx].bool()] = llm_positions.to(position_ids.device)+ else:+ position_ids[:, batch_idx] = llm_positions.to(position_ids.device)+ mrope_position_deltas.append(llm_positions.max() + 1 - len(current_input_ids))+ mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1)+ return position_ids, mrope_position_deltas++ @can_return_tuple+ @auto_docstring+ def get_video_features(+ self,+ pixel_values_videos: torch.FloatTensor,+ video_grid_thw: torch.LongTensor | None = None,+ **kwargs: Unpack[TransformersKwargs],+ ) -> tuple | BaseModelOutputWithPooling:+ r"""+ pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):+ The tensors corresponding to the input videos.+ video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):+ The temporal, height and width of feature shape of each video in LLM.+ """+ # Same implementation as for images+ kwargs.pop("return_dict", None)+ return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs)++ @can_return_tuple+ @auto_docstring+ def get_image_features(+ self,+ pixel_values: torch.FloatTensor,+ image_grid_thw: torch.LongTensor | None = None,+ **kwargs: Unpack[TransformersKwargs],+ ) -> tuple | BaseModelOutputWithPooling:+ r"""+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):+ The tensors corresponding to the input images.+ image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):+ The temporal, height and width of feature shape of each image in LLM.+ """+ kwargs.pop("return_dict", None)+ pixel_values = pixel_values.type(self.visual.dtype)+ vision_output: BaseModelOutputWithPooling = self.visual(+ pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs+ )+ image_embeds = vision_output.pooler_output+ split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()+ image_embeds = torch.split(image_embeds, split_sizes)+ vision_output.pooler_output = image_embeds++ return vision_output++ def get_placeholder_mask(+ self,+ input_ids: torch.LongTensor,+ inputs_embeds: torch.FloatTensor,+ image_features: torch.FloatTensor | None = None,+ video_features: torch.FloatTensor | None = None,+ ):+ """+ Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is+ equal to the length of multimodal features. If the lengths are different, an error is raised.+ """+ if input_ids is None:+ special_image_mask = inputs_embeds == self.get_input_embeddings()(+ torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)+ )+ special_image_mask = special_image_mask.all(-1)+ special_video_mask = inputs_embeds == self.get_input_embeddings()(+ torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)+ )+ special_video_mask = special_video_mask.all(-1)+ else:+ special_image_mask = input_ids == self.config.image_token_id+ special_video_mask = input_ids == self.config.video_token_id++ n_image_tokens = special_image_mask.sum()+ special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)+ if image_features is not None:+ torch_compilable_check(+ inputs_embeds[special_image_mask].numel() == image_features.numel(),+ f"Image features and image tokens do not match, tokens: {n_image_tokens}, features: {image_features.shape[0]}",+ )++ n_video_tokens = special_video_mask.sum()+ special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)+ if video_features is not None:+ torch_compilable_check(+ inputs_embeds[special_video_mask].numel() == video_features.numel(),+ f"Video features and video tokens do not match, tokens: {n_video_tokens}, features: {video_features.shape[0]}",+ )+ return special_image_mask, special_video_mask++ def compute_3d_position_ids(+ self,+ input_ids: torch.Tensor | None,+ inputs_embeds: torch.Tensor | None,+ image_grid_thw: torch.Tensor | None = None,+ video_grid_thw: torch.Tensor | None = None,+ attention_mask: torch.Tensor | None = None,+ past_key_values: torch.Tensor | None = None,+ mm_token_type_ids: torch.IntTensor | None = None,+ ) -> torch.Tensor | None:+ past_key_values_length = 0 if past_key_values is None else past_key_values.get_seq_length()+ can_compute_mrope = (+ input_ids is not None+ and mm_token_type_ids is not None+ and (image_grid_thw is not None or video_grid_thw is not None)+ )++ if can_compute_mrope and (self.rope_deltas is None or past_key_values_length == 0):+ position_ids, rope_deltas = self.get_rope_index(+ input_ids,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ attention_mask=attention_mask,+ mm_token_type_ids=mm_token_type_ids,+ )+ self.rope_deltas = rope_deltas+ # Use pre-calculated rope-deltas to infer correct 3D position ids+ elif self.rope_deltas is not None:+ batch_size, seq_length, _ = inputs_embeds.shape+ if attention_mask is not None:+ position_ids = attention_mask.long().cumsum(-1) - 1+ position_ids = position_ids.masked_fill(attention_mask == 0, 0)+ position_ids = position_ids.view(1, batch_size, -1).repeat(3, 1, 1).to(inputs_embeds.device)+ else:+ position_ids = torch.arange(past_key_values_length, past_key_values_length + seq_length)+ position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1).to(inputs_embeds.device)+ delta = self.rope_deltas.repeat_interleave(batch_size // self.rope_deltas.shape[0], dim=0)+ position_ids = position_ids + delta.to(device=inputs_embeds.device)+ else:+ # Can't build correct 3D positions. Let the model infer it from `cache_position`+ position_ids = None+ return position_ids++ @auto_docstring+ @can_return_tuple+ def forward(+ self,+ input_ids: torch.LongTensor = None,+ attention_mask: torch.Tensor | None = None,+ position_ids: torch.LongTensor | None = None,+ past_key_values: Cache | None = None,+ inputs_embeds: torch.FloatTensor | None = None,+ pixel_values: torch.Tensor | None = None,+ pixel_values_videos: torch.FloatTensor | None = None,+ image_grid_thw: torch.LongTensor | None = None,+ video_grid_thw: torch.LongTensor | None = None,+ mm_token_type_ids: torch.IntTensor | None = None,+ **kwargs: Unpack[TransformersKwargs],+ ) -> tuple | Qwen3_5ModelOutputWithPast:+ r"""+ image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):+ The temporal, height and width of feature shape of each image in LLM.+ video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):+ The temporal, height and width of feature shape of each video in LLM.+ """+ if (input_ids is None) ^ (inputs_embeds is not None):+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")++ # vision tower moved before text embedding to avoif flip / flop offloadling+ if pixel_values is not None:+ image_outputs: BaseModelOutputWithPooling = self.get_image_features(+ pixel_values, image_grid_thw, return_dict=True+ )+ image_embeds = image_outputs.pooler_output++ if pixel_values_videos is not None:+ video_outputs: BaseModelOutputWithPooling = self.get_video_features(+ pixel_values_videos, video_grid_thw, return_dict=True+ )+ video_embeds = video_outputs.pooler_output++ if inputs_embeds is None:+ inputs_embeds = self.get_input_embeddings()(input_ids)++ if pixel_values is not None:+ image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)+ image_mask, _ = self.get_placeholder_mask(+ input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds+ )+ inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)++ if pixel_values_videos is not None:+ video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)+ _, video_mask = self.get_placeholder_mask(+ input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds+ )+ inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)++ if position_ids is None:+ position_ids = self.compute_3d_position_ids(+ input_ids=input_ids,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ inputs_embeds=inputs_embeds,+ attention_mask=attention_mask,+ past_key_values=past_key_values,+ mm_token_type_ids=mm_token_type_ids,+ )++ outputs = self.language_model(+ input_ids=None,+ position_ids=position_ids,+ attention_mask=attention_mask,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ **kwargs,+ )++ return Qwen3_5ModelOutputWithPast(+ **outputs,+ rope_deltas=self.rope_deltas,+ )+++@auto_docstring+class Qwen3_5ForCausalLM(Qwen3_5PreTrainedModel, GenerationMixin):+ _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}+ _tp_plan = {"lm_head": "colwise_gather_output"}+ _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}+ config: Qwen3_5TextConfig+ _keys_to_ignore_on_load_unexpected = [r"^mtp.*", r"^model.visual.*"]++ def __init__(self, config):+ super().__init__(config)+ self.model = Qwen3_5TextModel(config)+ self.vocab_size = config.vocab_size+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)++ # Initialize weights and apply final processing+ self.post_init()++ @can_return_tuple+ @auto_docstring+ def forward(+ self,+ input_ids: torch.LongTensor | None = None,+ attention_mask: torch.Tensor | None = None,+ position_ids: torch.LongTensor | None = None,+ past_key_values: Cache | None = None,+ inputs_embeds: torch.FloatTensor | None = None,+ labels: torch.LongTensor | None = None,+ use_cache: bool | None = None,+ logits_to_keep: int | torch.Tensor = 0,+ **kwargs: Unpack[TransformersKwargs],+ ) -> CausalLMOutputWithPast:+ r"""+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.++ Example:++ ```python+ >>> from transformers import AutoTokenizer, Qwen3_5ForCausalLM++ >>> model = Qwen3_5ForCausalLM.from_pretrained("Qwen/Qwen3_5-8B")+ >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3_5-8B")++ >>> prompt = "Hey, are you conscious? Can you talk to me?"+ >>> inputs = tokenizer(prompt, return_tensors="pt")++ >>> # Generate+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]+ "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."+ ```"""+ outputs: BaseModelOutputWithPast = self.model(+ input_ids=input_ids,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ use_cache=use_cache,+ **kwargs,+ )++ hidden_states = outputs.last_hidden_state+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss+ slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep+ logits = self.lm_head(hidden_states[:, slice_indices, :])++ loss = None+ if labels is not None:+ loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)++ return CausalLMOutputWithPast(+ loss=loss,+ logits=logits,+ past_key_values=outputs.past_key_values,+ hidden_states=outputs.hidden_states,+ attentions=outputs.attentions,+ )+++class Qwen3_5ForSequenceClassification(GenericForSequenceClassification, Qwen3_5PreTrainedModel):+ config: Qwen3_5TextConfig+++@dataclass+@auto_docstring(+ custom_intro="""+ Base class for Qwen3_5 causal language model (or autoregressive) outputs.+ """+)+class Qwen3_5CausalLMOutputWithPast(ModelOutput):+ r"""+ loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):+ Language modeling loss (for next-token prediction).+ logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):+ Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).+ past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):+ It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).++ Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see+ `past_key_values` input) to speed up sequential decoding.+ rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):+ The rope index difference between sequence length and multimodal rope.+ """++ loss: torch.FloatTensor | None = None+ logits: torch.FloatTensor | None = None+ past_key_values: Cache | None = None+ hidden_states: tuple[torch.FloatTensor] | None = None+ attentions: tuple[torch.FloatTensor] | None = None+ rope_deltas: torch.LongTensor | None = None+++class Qwen3_5ForConditionalGeneration(Qwen3_5PreTrainedModel, GenerationMixin):+ _checkpoint_conversion_mapping = {}+ _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"}+ # Reference: fix gemma3 grad acc #37208+ accepts_loss_kwargs = False+ config: Qwen3_5Config++ def __init__(self, config):+ super().__init__(config)+ self.model = Qwen3_5Model(config)+ self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)++ self.post_init()++ def get_input_embeddings(self):+ return self.model.get_input_embeddings()++ def set_input_embeddings(self, value):+ self.model.set_input_embeddings(value)++ @auto_docstring+ def get_video_features(+ self,+ pixel_values_videos: torch.FloatTensor,+ video_grid_thw: torch.LongTensor | None = None,+ **kwargs: Unpack[TransformersKwargs],+ ) -> tuple | BaseModelOutputWithPooling:+ r"""+ pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):+ The tensors corresponding to the input videos.+ video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):+ The temporal, height and width of feature shape of each video in LLM.+ """+ return self.model.get_video_features(+ pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, **kwargs+ )++ @auto_docstring+ def get_image_features(+ self,+ pixel_values: torch.FloatTensor,+ image_grid_thw: torch.LongTensor | None = None,+ **kwargs: Unpack[TransformersKwargs],+ ) -> tuple | BaseModelOutputWithPooling:+ r"""+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):+ The tensors corresponding to the input images.+ image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):+ The temporal, height and width of feature shape of each image in LLM.+ """+ return self.model.get_image_features(pixel_values=pixel_values, image_grid_thw=image_grid_thw, **kwargs)++ @can_return_tuple+ def forward(+ self,+ input_ids: torch.LongTensor = None,+ attention_mask: torch.Tensor | None = None,+ position_ids: torch.LongTensor | None = None,+ past_key_values: Cache | None = None,+ inputs_embeds: torch.FloatTensor | None = None,+ labels: torch.LongTensor | None = None,+ pixel_values: torch.Tensor | None = None,+ pixel_values_videos: torch.FloatTensor | None = None,+ image_grid_thw: torch.LongTensor | None = None,+ video_grid_thw: torch.LongTensor | None = None,+ mm_token_type_ids: torch.IntTensor | None = None,+ logits_to_keep: int | torch.Tensor = 0,+ **kwargs: Unpack[TransformersKwargs],+ ) -> tuple | Qwen3_5CausalLMOutputWithPast:+ r"""+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.+ image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):+ The temporal, height and width of feature shape of each image in LLM.+ video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):+ The temporal, height and width of feature shape of each video in LLM.++ Example:++ ```python+ >>> from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration++ >>> model = Qwen3_5ForConditionalGeneration.from_pretrained("Qwen/Qwen3-VL-8B-Instruct")+ >>> processor = AutoProcessor.from_pretrained("Qwen/Qwen3-VL-8B-Instruct")++ >>> messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "image",+ "image": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg",+ },+ {"type": "text", "text": "Describe the image."},+ ],+ }+ ]++ >>> inputs = processor.apply_chat_template(+ messages,+ tokenize=True,+ add_generation_prompt=True,+ return_dict=True,+ return_tensors="pt"+ )++ >>> # Generate+ >>> generated_ids = model.generate(**inputs, max_new_tokens=1024)+ >>> generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]+ >>> output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]+ >>> print(output_text)+ ```+ """++ outputs = self.model(+ input_ids=input_ids,+ pixel_values=pixel_values,+ pixel_values_videos=pixel_values_videos,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ position_ids=position_ids,+ attention_mask=attention_mask,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ mm_token_type_ids=mm_token_type_ids,+ **kwargs,+ )++ hidden_states = outputs[0]++ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss+ slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep+ logits = self.lm_head(hidden_states[:, slice_indices, :])++ loss = None+ if labels is not None:+ loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size)++ return Qwen3_5CausalLMOutputWithPast(+ loss=loss,+ logits=logits,+ past_key_values=outputs.past_key_values,+ hidden_states=outputs.hidden_states,+ attentions=outputs.attentions,+ rope_deltas=outputs.rope_deltas,+ )++ def prepare_inputs_for_generation(+ self,+ input_ids,+ past_key_values=None,+ attention_mask=None,+ inputs_embeds=None,+ position_ids=None,+ use_cache=True,+ pixel_values=None,+ pixel_values_videos=None,+ image_grid_thw=None,+ video_grid_thw=None,+ is_first_iteration=False,+ **kwargs,+ ):+ # Overwritten -- in specific circumstances we don't want to forward image inputs to the model++ effective_first_iteration = bool(is_first_iteration)+ if not effective_first_iteration:+ if past_key_values is None:+ effective_first_iteration = True+ else:+ get_seq_length = getattr(past_key_values, "get_seq_length", None)+ if callable(get_seq_length):+ try:+ effective_first_iteration = int(get_seq_length()) == 0+ except Exception:+ effective_first_iteration = False++ model_inputs = super().prepare_inputs_for_generation(+ input_ids,+ past_key_values=past_key_values,+ attention_mask=attention_mask,+ inputs_embeds=inputs_embeds,+ position_ids=position_ids,+ pixel_values=pixel_values,+ pixel_values_videos=pixel_values_videos,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ use_cache=use_cache,+ is_first_iteration=effective_first_iteration,+ **kwargs,+ )++ if not effective_first_iteration and use_cache:+ model_inputs["pixel_values"] = None+ model_inputs["pixel_values_videos"] = None++ return model_inputs++ def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs):+ # Overwritten -- requires 3D position ids++ text_positions = super()._prepare_position_ids_for_generation(inputs_tensor, model_kwargs)++ # Early exit in case we are continuing generation from past kv+ past_length = 0+ if (cache := model_kwargs.get("past_key_values")) is not None:+ past_length = cache.get_seq_length()+ if past_length != 0 and self.model.rope_deltas is not None:+ position_ids = text_positions[None, ...] + self.model.rope_deltas+ return position_ids++ # Otherwise compute 3d position ids for vision tokens and concat with text position ids+ if "input_ids" in model_kwargs and model_kwargs["input_ids"].shape[1] > 0:+ inputs_tensor = model_kwargs["input_ids"]++ is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long]+ if (+ is_input_ids+ and model_kwargs.get("mm_token_type_ids") is not None+ and (model_kwargs.get("image_grid_thw") is not None or model_kwargs.get("video_grid_thw") is not None)+ ):+ model_kwargs = {k: v for k, v in model_kwargs.items() if k != "input_ids"}+ vision_positions, rope_deltas = self.model.get_rope_index(inputs_tensor, **model_kwargs)+ self.model.rope_deltas = rope_deltas+ else:+ vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1)+ self.model.rope_deltas = torch.zeros(+ inputs_tensor.shape[0], 1, dtype=torch.long, device=inputs_tensor.device+ )++ # Concatenate "text + vision" positions into [4, bs, seq-len]+ text_positions = text_positions[None, ...]+ position_ids = torch.cat([text_positions, vision_positions], dim=0)++ return position_ids++ def _get_image_nums_and_video_nums(+ self,+ input_ids: torch.LongTensor | None,+ inputs_embeds: torch.Tensor | None = None,+ ) -> tuple[torch.Tensor, torch.Tensor]:+ """+ Get the number of images and videos for each sample to calculate the separation length of the sample tensor.+ These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications.++ Args:+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):+ Indices of input sequence tokens in the vocabulary.++ Returns:+ image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`)+ video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`)+ """+ image_token_id = self.config.image_token_id+ video_token_id = self.config.video_token_id+ vision_start_token_id = self.config.vision_start_token_id++ if inputs_embeds is not None:+ vision_start_mask = (+ inputs_embeds+ == self.get_input_embeddings()(+ torch.tensor(vision_start_token_id, dtype=torch.long, device=inputs_embeds.device)+ )+ )[..., 0]+ image_mask = (+ inputs_embeds+ == self.get_input_embeddings()(+ torch.tensor(image_token_id, dtype=torch.long, device=inputs_embeds.device)+ )+ )[..., 0]+ video_mask = (+ inputs_embeds+ == self.get_input_embeddings()(+ torch.tensor(video_token_id, dtype=torch.long, device=inputs_embeds.device)+ )+ )[..., 0]+ else:+ vision_start_mask = input_ids == vision_start_token_id+ image_mask = input_ids == image_token_id+ video_mask = input_ids == video_token_id++ vision_first_mask = torch.roll(vision_start_mask, shifts=1, dims=1)+ image_nums = torch.sum(vision_first_mask & image_mask, dim=1)+ video_nums = torch.sum(vision_first_mask & video_mask, dim=1)++ return image_nums, video_nums++ def _expand_inputs_for_generation(+ self,+ expand_size: int = 1,+ is_encoder_decoder: bool = False,+ input_ids: torch.LongTensor | None = None,+ **model_kwargs,+ ) -> tuple[torch.LongTensor, dict[str, Any]]:+ # Overwritten -- Qwen3_5 use timestamps and remove second_per_grid_ts+ # Support for expanding tensors without a batch size dimension+ # e.g., pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw+ # pixel_values.shape[0] is sum(seqlen_images for samples)+ # image_grid_thw.shape[0] is sum(num_images for samples)++ if expand_size == 1:+ return input_ids, model_kwargs++ visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw"]++ def _expand_dict_for_generation_visual(dict_to_expand):+ image_grid_thw = model_kwargs.get("image_grid_thw", None)+ video_grid_thw = model_kwargs.get("video_grid_thw", None)+ image_nums, video_nums = self._get_image_nums_and_video_nums(+ input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None)+ )++ # video_nums: (batch_size,)+ # since video_nums is the number of videos in the input dependent on the input_ids(vision_start),+ # but Qwen3_5 append vision_start to each frame of each video, so we need to recover the real video_nums according to video_grid_thw+ if video_grid_thw is not None:+ cumulative_frame_counts = torch.cumsum(video_grid_thw[:, 0], dim=0)+ cumulative_token_video_counts = torch.cumsum(video_nums, dim=0)+ # Find video boundaries in cumulative_frame_counts+ video_boundary_indices = torch.searchsorted(cumulative_frame_counts, cumulative_token_video_counts)+ # example: video_boundary_indices = [3, 5] means video_nums = [4, 2]+ video_nums = torch.diff(torch.cat([-video_boundary_indices.new_ones(1), video_boundary_indices]))++ def _repeat_interleave_samples(x, lengths, repeat_times):+ samples = torch.split(x, lengths)+ repeat_args = [repeat_times] + [1] * (x.dim() - 1)+ result = torch.cat([sample.repeat(*repeat_args) for sample in samples], dim=0)+ return result++ for key in dict_to_expand:+ if key == "pixel_values":+ # split images into samples+ samples = torch.split(image_grid_thw, list(image_nums))+ # compute the sequence length of images for each sample+ lengths = [torch.prod(sample, dim=1).sum() for sample in samples]+ dict_to_expand[key] = _repeat_interleave_samples(+ dict_to_expand[key], lengths=lengths, repeat_times=expand_size+ )+ elif key == "image_grid_thw":+ # get the num of images for each sample+ lengths = list(image_nums)+ dict_to_expand[key] = _repeat_interleave_samples(+ dict_to_expand[key], lengths=lengths, repeat_times=expand_size+ )+ elif key == "pixel_values_videos":+ samples = torch.split(video_grid_thw, list(video_nums))+ lengths = [torch.prod(sample, dim=1).sum() for sample in samples]+ dict_to_expand[key] = _repeat_interleave_samples(+ dict_to_expand[key], lengths=lengths, repeat_times=expand_size+ )+ elif key == "video_grid_thw":+ lengths = list(video_nums)+ dict_to_expand[key] = _repeat_interleave_samples(+ dict_to_expand[key], lengths=lengths, repeat_times=expand_size+ )+ return dict_to_expand++ def _expand_dict_for_generation(dict_to_expand):+ for key in dict_to_expand:+ if key == "position_ids" and dict_to_expand[key].ndim == 3:+ dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=1)+ elif (+ key != "cache_position"+ and dict_to_expand[key] is not None+ and isinstance(dict_to_expand[key], torch.Tensor)+ and key not in visual_keys+ ):+ dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=0)+ return dict_to_expand++ model_kwargs = _expand_dict_for_generation_visual(model_kwargs)++ if input_ids is not None:+ input_ids = input_ids.repeat_interleave(expand_size, dim=0)++ model_kwargs = _expand_dict_for_generation(model_kwargs)++ if is_encoder_decoder:+ if model_kwargs.get("encoder_outputs") is None:+ raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.")+ model_kwargs["encoder_outputs"] = _expand_dict_for_generation(model_kwargs["encoder_outputs"])++ return input_ids, model_kwargs+++__all__ = [+ "Qwen3_5VisionModel",+ "Qwen3_5TextModel",+ "Qwen3_5Model",+ "Qwen3_5ForCausalLM",+ "Qwen3_5ForSequenceClassification",+ "Qwen3_5ForConditionalGeneration",+ "Qwen3_5PreTrainedModel",+]diff --git a/shared/qtypes/gguf.py b/shared/qtypes/gguf.pyindex 33569b2..6544b5f 100644--- a/shared/qtypes/gguf.py+++ b/shared/qtypes/gguf.py@@ -5,10 +5,11 @@ import reimport torchfrom torch.utils import _pytree as pytree-from optimum.quanto import QModuleMixin+from optimum.quanto import QModuleMixin, register_qmodulefrom optimum.quanto.tensor.qtensor import QTensorfrom optimum.quanto.tensor.qtype import qtype as _quanto_qtype, qtypes as _quanto_qtypesfrom collections import OrderedDict+from mmgp.offload import QEmbedding as BaseQEmbeddingHANDLER_NAME = "gguf"@@ -33,6 +34,11 @@ _GGUF_QTYPE = _quanto_qtypes[_GGUF_QTYPE_NAME]_GGUF_DEFAULT_DTYPE = None_GGUF_LABEL_CACHE = {}+_GGUF_RUNTIME_LOGGED = set()+_GGUF_CUDA_KERNELS_ENV = "WGP_GGUF_LLAMACPP_CUDA"+_GGUF_CUDA_KERNELS_ENABLED_CACHE = None+_GGUF_CUDA_MODULE = None+_GGUF_CUDA_LOAD_ERROR = Nonedef _normalize_gguf_path(file_path):@@ -60,6 +66,97 @@ def _resolve_default_dtype(dtype, fallback=None):return _GGUF_DEFAULT_DTYPEreturn dtype++def _gguf_log_once(key, message):+ if key in _GGUF_RUNTIME_LOGGED:+ return+ _GGUF_RUNTIME_LOGGED.add(key)+ print(message)+++def _gguf_cuda_env_enabled():+ raw = str(os.environ.get(_GGUF_CUDA_KERNELS_ENV, "1")).strip().lower()+ return raw in ("1", "true", "yes", "y", "on")+++def _validate_gguf_cuda_module(module):+ import numpy as np++ if gguf is None:+ raise RuntimeError("gguf package is unavailable")+ if not torch.cuda.is_available():+ raise RuntimeError("CUDA is not available")+ if not module.may_support_linear_qtype_name("Q4_0"):+ raise RuntimeError("Q4_0 linear is not supported by the installed GGUF CUDA package")++ weight = np.linspace(-1.0, 1.0, 32, dtype=np.float32).reshape(1, 32)+ quantized_weight = gguf.quants.quantize(weight, gguf.GGMLQuantizationType.Q4_0)+ raw_weight = torch.from_numpy(quantized_weight).contiguous().cuda()+ input_tensor = torch.linspace(-0.5, 0.5, 64, dtype=torch.float32, device="cuda").reshape(2, 32)+ fast_out = module.linear(raw_weight, "Q4_0", [1, 32], input_tensor, None, torch.float32)+ dense_weight = torch.from_numpy(gguf.quants.dequantize(quantized_weight, gguf.GGMLQuantizationType.Q4_0)).to(device=input_tensor.device, dtype=torch.float32)+ ref_out = torch.nn.functional.linear(input_tensor, dense_weight, None)+ torch.cuda.synchronize()+ if fast_out.shape != ref_out.shape:+ raise RuntimeError(f"Q4 self-test shape mismatch: {tuple(fast_out.shape)} vs {tuple(ref_out.shape)}")+ if not torch.allclose(fast_out, ref_out, atol=2e-2, rtol=2e-2):+ max_diff = float((fast_out - ref_out).abs().max().item())+ raise RuntimeError(f"Q4 self-test output mismatch (max diff {max_diff:.6f})")+++def _probe_gguf_cuda_runtime(force=False):+ global _GGUF_CUDA_KERNELS_ENABLED_CACHE, _GGUF_CUDA_MODULE, _GGUF_CUDA_LOAD_ERROR+ if not force and not _gguf_cuda_env_enabled():+ _GGUF_CUDA_KERNELS_ENABLED_CACHE = False+ _GGUF_CUDA_MODULE = None+ _GGUF_CUDA_LOAD_ERROR = None+ return False+ try:+ import llamacpp_gguf_cuda as gguf_cuda_module+ _validate_gguf_cuda_module(gguf_cuda_module)+ except Exception as exc:+ _GGUF_CUDA_KERNELS_ENABLED_CACHE = False+ _GGUF_CUDA_MODULE = None+ _GGUF_CUDA_LOAD_ERROR = exc+ _gguf_log_once("gguf_cuda_probe_failed", f"[GGUF][llama.cpp CUDA] kernels unavailable, using fallback: {exc}")+ return False+ _GGUF_CUDA_KERNELS_ENABLED_CACHE = True+ _GGUF_CUDA_MODULE = gguf_cuda_module+ _GGUF_CUDA_LOAD_ERROR = None+ _gguf_log_once("gguf_cuda_probe_ok", "[GGUF][llama.cpp CUDA] kernels available.")+ return True+++def _gguf_cuda_kernels_enabled():+ global _GGUF_CUDA_KERNELS_ENABLED_CACHE+ if _GGUF_CUDA_KERNELS_ENABLED_CACHE is not None:+ return _GGUF_CUDA_KERNELS_ENABLED_CACHE+ return _probe_gguf_cuda_runtime()+++def _gguf_cuda_module():+ if not _gguf_cuda_kernels_enabled():+ return None+ return _GGUF_CUDA_MODULE+++def set_gguf_cuda_kernels_enabled(enabled=None):+ global _GGUF_CUDA_KERNELS_ENABLED_CACHE, _GGUF_CUDA_MODULE, _GGUF_CUDA_LOAD_ERROR+ if enabled is False:+ _GGUF_CUDA_KERNELS_ENABLED_CACHE = False+ _GGUF_CUDA_MODULE = None+ _GGUF_CUDA_LOAD_ERROR = None+ return+ _GGUF_CUDA_KERNELS_ENABLED_CACHE = None+ _GGUF_CUDA_MODULE = None+ _GGUF_CUDA_LOAD_ERROR = None+ if enabled is None:+ return _probe_gguf_cuda_runtime()+ return _probe_gguf_cuda_runtime(force=True)+++_probe_gguf_cuda_runtime()+def get_file_metadata(file_path):if gguf is None:raise RuntimeError("GGUF support requires the 'gguf' package.")@@ -290,6 +387,39 @@ class GGUFSourceTensor(torch.Tensor):torch.utils.swap_tensors(self, data)+def materialize_source_tensor(tensor: torch.Tensor) -> torch.Tensor:+ if not isinstance(tensor, GGUFSourceTensor):+ return tensor+ with torch._C.DisableTorchFunctionSubclass():+ plain = torch.empty_strided(+ tuple(tensor.shape),+ tuple(tensor.stride()),+ dtype=tensor.dtype,+ device=tensor.device,+ )+ plain.copy_(tensor)+ return plain+++def materialize_module_source_tensors(module: torch.nn.Module) -> int:+ converted = 0+ for submodule in module.modules():+ for name, param in list(submodule._parameters.items()):+ if param is None or not isinstance(param, GGUFSourceTensor):+ continue+ submodule._parameters[name] = torch.nn.Parameter(+ materialize_source_tensor(param),+ requires_grad=param.requires_grad,+ )+ converted += 1+ for name, buf in list(submodule._buffers.items()):+ if buf is None or not isinstance(buf, GGUFSourceTensor):+ continue+ submodule._buffers[name] = materialize_source_tensor(buf)+ converted += 1+ return converted++def _split_gguf_tensor(src, *, dim, split_sizes, context):if not torch.is_tensor(src):return None@@ -345,6 +475,76 @@ def _gguf_qtype_name(qtype_obj):return getattr(qtype_obj, "name", None) or str(qtype_obj)+def _try_llamacpp_cuda_linear(weight_tensor, input_tensor, bias, target_dtype):+ if not _gguf_cuda_kernels_enabled():+ return None+ if not torch.is_tensor(input_tensor) or input_tensor.device.type != "cuda":+ return None+ raw = getattr(weight_tensor, "_data", None)+ if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():+ return None+ qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))+ try:+ gguf_llamacpp_cuda = _gguf_cuda_module()+ if gguf_llamacpp_cuda is None or not gguf_llamacpp_cuda.may_support_linear_qtype_name(qtype_name):+ return None+ return gguf_llamacpp_cuda.linear(raw, qtype_name, tuple(getattr(weight_tensor, "_tensor_shape", weight_tensor.shape)), input_tensor, bias, target_dtype)+ except Exception as exc:+ _gguf_log_once(f"llamacpp_cuda_linear_{qtype_name}", f"[GGUF][llama.cpp CUDA] linear GGUF CUDA kernels failed for {qtype_name}, using fallback: {exc}")+ return None+++def _try_llamacpp_cuda_embedding(weight_tensor, index_tensor, target_dtype):+ if not _gguf_cuda_kernels_enabled():+ return None+ if not torch.is_tensor(index_tensor) or index_tensor.device.type != "cuda":+ return None+ raw = getattr(weight_tensor, "_data", None)+ if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():+ return None+ qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))+ try:+ gguf_llamacpp_cuda = _gguf_cuda_module()+ if gguf_llamacpp_cuda is None or not gguf_llamacpp_cuda.may_support_embedding_qtype_name(qtype_name):+ return None+ return gguf_llamacpp_cuda.embedding(raw, qtype_name, tuple(getattr(weight_tensor, "_tensor_shape", weight_tensor.shape)), index_tensor, target_dtype)+ except Exception as exc:+ _gguf_log_once(f"llamacpp_cuda_embedding_{qtype_name}", f"[GGUF][llama.cpp CUDA] embedding GGUF CUDA kernels failed for {qtype_name}, using fallback: {exc}")+ return None+++def _may_try_llamacpp_cuda_linear(weight_tensor, input_tensor):+ if not _gguf_cuda_kernels_enabled():+ return False+ if not torch.is_tensor(input_tensor) or input_tensor.device.type != "cuda":+ return False+ raw = getattr(weight_tensor, "_data", None)+ if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():+ return False+ qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))+ try:+ gguf_llamacpp_cuda = _gguf_cuda_module()+ return gguf_llamacpp_cuda is not None and gguf_llamacpp_cuda.may_support_linear_qtype_name(qtype_name)+ except Exception:+ return False+++def _may_try_llamacpp_cuda_embedding(weight_tensor, index_tensor):+ if not _gguf_cuda_kernels_enabled():+ return False+ if not torch.is_tensor(index_tensor) or index_tensor.device.type != "cuda":+ return False+ raw = getattr(weight_tensor, "_data", None)+ if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():+ return False+ qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))+ try:+ gguf_llamacpp_cuda = _gguf_cuda_module()+ return gguf_llamacpp_cuda is not None and gguf_llamacpp_cuda.may_support_embedding_qtype_name(qtype_name)+ except Exception:+ return False++def _guess_variant_from_filename(filename):base = os.path.basename(str(filename))match = re.search(r"(?i)(?:^|[_-])(Q\d+_K|Q\d+_\d|Q\d+|IQ\d+_\w+)(?:$|[_.-])", base)@@ -725,6 +925,10 @@ class GGUFWeightTensor(QTensor):else:target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=self.dtype)target_device = self.device+ if _may_try_llamacpp_cuda_linear(self, input):+ fast_out = _try_llamacpp_cuda_linear(self, input, bias, target_dtype)+ if fast_out is not None:+ return fast_outweight = self.dequantize(dtype=target_dtype, device=target_device)if torch.is_tensor(input) and input.dtype != weight.dtype:input = input.to(weight.dtype)@@ -970,6 +1174,267 @@ class QLinearGGUF(QModuleMixin, torch.nn.Linear):return+@register_qmodule(torch.nn.Conv1d)+class QConv1dGGUF(QModuleMixin, torch.nn.Conv1d):+ def __init__(+ self,+ in_channels,+ out_channels,+ kernel_size,+ stride=1,+ padding=0,+ dilation=1,+ groups=1,+ bias=True,+ padding_mode="zeros",+ device=None,+ dtype=None,+ weights=None,+ activations=None,+ optimizer=None,+ quantize_input=True,+ ):+ super().__init__(+ in_channels,+ out_channels,+ kernel_size,+ stride=stride,+ padding=padding,+ dilation=dilation,+ groups=groups,+ bias=bias,+ padding_mode=padding_mode,+ device=device,+ dtype=dtype,+ weights=weights,+ activations=activations,+ optimizer=optimizer,+ quantize_input=quantize_input,+ )+ self._gguf_default_dtype = dtype++ @classmethod+ def qcreate(cls, module, weights, activations=None, optimizer=None, device=None):+ if torch.is_tensor(module.weight) and module.weight.dtype.is_floating_point:+ weight_dtype = module.weight.dtype+ elif torch.is_tensor(getattr(module, "bias", None)) and module.bias.dtype.is_floating_point:+ weight_dtype = module.bias.dtype+ else:+ weight_dtype = torch.float16+ return cls(+ module.in_channels,+ module.out_channels,+ module.kernel_size,+ stride=module.stride,+ padding=module.padding,+ dilation=module.dilation,+ groups=module.groups,+ bias=module.bias is not None,+ padding_mode=module.padding_mode,+ device=device,+ dtype=weight_dtype,+ weights=weights,+ activations=activations,+ optimizer=optimizer,+ quantize_input=True,+ )++ def set_default_dtype(self, dtype):+ self._gguf_default_dtype = dtype++ @property+ def qweight(self):+ if self.weight_qtype == _GGUF_QTYPE:+ return self.weight+ return super().qweight++ def forward(self, input: torch.Tensor) -> torch.Tensor:+ qweight = self.qweight+ if isinstance(qweight, GGUFWeightTensor):+ target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=input.dtype)+ weight = qweight.dequantize(dtype=target_dtype, device=input.device)+ bias = _maybe_cast_bias(self.bias, weight.dtype)+ if input.dtype != weight.dtype:+ input = input.to(weight.dtype)+ return torch.nn.functional.conv1d(+ input,+ weight,+ bias=bias,+ stride=self.stride,+ padding=self.padding,+ dilation=self.dilation,+ groups=self.groups,+ )+ return torch.nn.functional.conv1d(+ input,+ qweight,+ bias=self.bias,+ stride=self.stride,+ padding=self.padding,+ dilation=self.dilation,+ groups=self.groups,+ )++ def _load_from_state_dict(+ self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ ):+ if self.weight_qtype != _GGUF_QTYPE:+ return super()._load_from_state_dict(+ state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ )++ weight_key = prefix + "weight"+ bias_key = prefix + "bias"+ input_scale_key = prefix + "input_scale"+ output_scale_key = prefix + "output_scale"++ weight_raw = state_dict.pop(weight_key, None)+ bias = state_dict.pop(bias_key, None)+ input_scale = state_dict.pop(input_scale_key, None)+ output_scale = state_dict.pop(output_scale_key, None)++ if weight_raw is None:+ missing_keys.append(weight_key)++ target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=self.weight.dtype)+ if weight_raw is not None:+ gguf_weight = GGUFWeightTensor.create(+ raw_tensor=weight_raw,+ size=self.weight.size(),+ stride=self.weight.stride(),+ dtype=target_dtype,+ device=weight_raw.device,+ requires_grad=False,+ )+ self.weight = torch.nn.Parameter(gguf_weight, requires_grad=False)++ if bias is not None:+ self.bias = torch.nn.Parameter(bias, requires_grad=False)++ if torch.is_tensor(weight_raw):+ scale_device = weight_raw.device+ elif torch.is_tensor(self.weight):+ scale_device = self.weight.device+ elif torch.is_tensor(bias):+ scale_device = bias.device+ else:+ scale_device = torch.device("cpu")++ if input_scale is not None:+ self.input_scale = input_scale.to(scale_device)+ elif not hasattr(self, "input_scale") or self.input_scale.is_meta:+ scale_dtype = self.input_scale.dtype if hasattr(self, "input_scale") else torch.float32+ self.input_scale = torch.ones((), dtype=scale_dtype, device=scale_device)++ if output_scale is not None:+ self.output_scale = output_scale.to(scale_device)+ elif not hasattr(self, "output_scale") or self.output_scale.is_meta:+ scale_dtype = self.output_scale.dtype if hasattr(self, "output_scale") else torch.float32+ self.output_scale = torch.ones((), dtype=scale_dtype, device=scale_device)++ return+++@register_qmodule(torch.nn.Embedding)+class QEmbeddingGGUF(BaseQEmbedding):+ def __init__(+ self,+ num_embeddings,+ embedding_dim,+ padding_idx=None,+ max_norm=None,+ norm_type=2.0,+ scale_grad_by_freq=False,+ sparse=False,+ device=None,+ dtype=None,+ weights=None,+ activations=None,+ optimizer=None,+ quantize_input=True,+ ):+ super().__init__(+ num_embeddings,+ embedding_dim,+ padding_idx=padding_idx,+ max_norm=max_norm,+ norm_type=norm_type,+ scale_grad_by_freq=scale_grad_by_freq,+ sparse=sparse,+ device=device,+ dtype=dtype,+ weights=weights,+ activations=activations,+ optimizer=optimizer,+ quantize_input=quantize_input,+ )+ self._gguf_default_dtype = dtype++ def set_default_dtype(self, dtype):+ self._gguf_default_dtype = dtype++ @property+ def qweight(self):+ if self.weight_qtype == _GGUF_QTYPE:+ return self.weight+ return super().qweight++ def forward(self, input: torch.Tensor) -> torch.Tensor:+ qweight = self.qweight+ if isinstance(qweight, GGUFWeightTensor):+ target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=torch.float16)+ if self.max_norm is None and not self.scale_grad_by_freq and not self.sparse:+ if _may_try_llamacpp_cuda_embedding(qweight, input):+ fast_out = _try_llamacpp_cuda_embedding(qweight, input, target_dtype)+ if fast_out is not None:+ return fast_out+ weight = qweight.dequantize(dtype=target_dtype, device=input.device)+ return torch.nn.functional.embedding(+ input,+ weight,+ self.padding_idx,+ self.max_norm,+ self.norm_type,+ self.scale_grad_by_freq,+ self.sparse,+ )+ return super().forward(input)++ def _load_from_state_dict(+ self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ ):+ if self.weight_qtype != _GGUF_QTYPE:+ return super()._load_from_state_dict(+ state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ )++ weight_key = prefix + "weight"+ weight_raw = state_dict.pop(weight_key, None)+ if weight_raw is None:+ missing_keys.append(weight_key)+ return++ target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=self.weight.dtype)+ gguf_weight = GGUFWeightTensor.create(+ raw_tensor=weight_raw,+ size=self.weight.size(),+ stride=self.weight.stride(),+ dtype=target_dtype,+ device=weight_raw.device,+ requires_grad=False,+ )+ self.weight = torch.nn.Parameter(gguf_weight, requires_grad=False)+ scale_device = weight_raw.device if torch.is_tensor(weight_raw) else torch.device("cpu")+ if not hasattr(self, "input_scale") or self.input_scale is None or self.input_scale.is_meta:+ self.input_scale = torch.ones((), dtype=torch.float32, device=scale_device)+ else:+ self.input_scale = self.input_scale.to(scale_device)+ if not hasattr(self, "output_scale") or self.output_scale is None or self.output_scale.is_meta:+ self.output_scale = torch.ones((), dtype=torch.float32, device=scale_device)+ else:+ self.output_scale = self.output_scale.to(scale_device)++def _collect_gguf_specs(state_dict):specs = []for key, tensor in state_dict.items():diff --git a/shared/utils/audio_video.py b/shared/utils/audio_video.pyindex 84110bb..b8e225a 100644--- a/shared/utils/audio_video.py+++ b/shared/utils/audio_video.py@@ -104,6 +104,18 @@ def _get_audio_codec_settings(codec_key):return settings.get(codec_key, settings["wav"])+def get_mp4_audio_codec_settings(codec_key):+ codec_key = "aac_128" if not codec_key else str(codec_key).lower()+ settings = {+ "aac_128": {"codec": "aac", "bitrate": "128k", "ext": ".aac"},+ "aac_192": {"codec": "aac", "bitrate": "192k", "ext": ".aac"},+ "aac_256": {"codec": "aac", "bitrate": "256k", "ext": ".aac"},+ "aac_320": {"codec": "aac", "bitrate": "320k", "ext": ".aac"},+ "alac": {"codec": "alac", "bitrate": None, "ext": ".m4a"},+ }+ return settings.get(codec_key, settings["aac_128"])++def get_audio_codec_extension(codec_key):return _get_audio_codec_settings(codec_key)["ext"]@@ -151,9 +163,9 @@ def extract_audio_track_to_wav(video_path, output_path):-def extract_audio_tracks(source_video, verbose=False, query_only=False):+def extract_audio_tracks(source_video, verbose=False, query_only=False, codec_key="aac_128", temp_format=None):"""- Extract all audio tracks from a source video into temporary AAC files.+ Extract all audio tracks from a source video into temporary audio files.Returns:Tuple:@@ -195,9 +207,13 @@ def extract_audio_tracks(source_video, verbose=False, query_only=False):file_paths = []metadata = []+ if temp_format == "wav":+ audio_settings = {"codec": "pcm_s16le", "bitrate": None, "ext": ".wav"}+ else:+ audio_settings = get_mp4_audio_codec_settings(codec_key)for i, stream in enumerate(audio_streams):- fd, temp_path = tempfile.mkstemp(suffix=f'_track{i}.aac', prefix='audio_')+ fd, temp_path = tempfile.mkstemp(suffix=f'_track{i}{audio_settings["ext"]}', prefix='audio_')os.close(fd)file_paths.append(temp_path)@@ -209,10 +225,10 @@ def extract_audio_tracks(source_video, verbose=False, query_only=False):'language': stream.get('tags', {}).get('language', None)})- ffmpeg.input(source_video).output(- temp_path,- **{f'map': f'0:a:{i}', 'acodec': 'aac', 'b:a': '128k'}- ).overwrite_output().run(quiet=not verbose)+ output_kwargs = {f'map': f'0:a:{i}', 'acodec': audio_settings["codec"]}+ if audio_settings["bitrate"]:+ output_kwargs['b:a'] = audio_settings["bitrate"]+ ffmpeg.input(source_video).output(temp_path, **output_kwargs).overwrite_output().run(quiet=not verbose)return file_paths, metadata@@ -224,10 +240,12 @@ def combine_and_concatenate_video_with_audio_tracks(source_audio_duration, audio_sampling_rate,new_audio_from_start=False,source_audio_metadata=None,- audio_bitrate='128k',- audio_codec='aac',+ audio_codec_key="aac_128",verbose = False):+ audio_settings = get_mp4_audio_codec_settings(audio_codec_key)+ audio_codec = audio_settings["codec"]+ audio_bitrate = audio_settings["bitrate"]inputs, filters, maps, idx = ['-i', video_path], [], ['-map', '0:v'], 1metadata_args = []sources = source_audio_tracks or []@@ -290,10 +308,11 @@ def combine_and_concatenate_video_with_audio_tracks(*maps, *metadata_args,'-c:v', 'copy','-c:a', audio_codec,- '-b:a', audio_bitrate,'-ar', str(audio_sampling_rate),'-ac', '1','-shortest', save_path_tmp]+ if audio_bitrate:+ cmd[-6:-6] = ['-b:a', audio_bitrate]if verbose:print(f"ffmpeg command: {cmd}")diff --git a/shared/utils/files_locator.py b/shared/utils/files_locator.pyindex 8d4bec4..10d73ed 100644--- a/shared/utils/files_locator.py+++ b/shared/utils/files_locator.py@@ -12,6 +12,18 @@ def set_checkpoints_paths(checkpoints_paths):_checkpoints_paths = [path.strip() for path in checkpoints_paths if len(path.strip()) > 0 ]if len(checkpoints_paths) == 0:_checkpoints_paths = default_checkpoints_paths++def _normalize_force_path(force_path):+ if force_path is not None and isinstance(force_path, list) and len(force_path):+ force_path = force_path[0]+ if force_path is None:+ return None+ force_path = os.fspath(force_path).strip()+ if len(force_path) == 0:+ return None+ normalized = os.path.normpath(force_path)+ return None if normalized in ("", ".") else normalized+def get_download_location(file_name = None, force_path= None):if file_name is not None and os.path.isabs(file_name): return file_nameif force_path is not None and isinstance(force_path, list) and len(force_path): force_path = force_path[0]@@ -26,6 +38,30 @@ def get_download_location(file_name = None, force_path= None):else:return os.path.join(_checkpoints_paths[0])+def get_smart_download_root(force_path = None):+ force_path = _normalize_force_path(force_path)+ if force_path is None:+ return _checkpoints_paths[0]+ if os.path.isabs(force_path):+ return force_path+ for folder in _checkpoints_paths:+ candidate = os.path.join(folder, force_path)+ if os.path.isdir(candidate):+ return folder+ return _checkpoints_paths[0]++def get_smart_download_location(file_name = None, force_path = None):+ if file_name is not None and os.path.isabs(file_name):+ return file_name+ force_path = _normalize_force_path(force_path)+ if force_path is None:+ return get_download_location(file_name)+ if os.path.isabs(force_path):+ return force_path if file_name is None else os.path.join(force_path, file_name)+ root = get_smart_download_root(force_path)+ base_path = os.path.join(root, force_path)+ return base_path if file_name is None else os.path.join(base_path, file_name)+def locate_folder(folder_name, error_if_none = True):searched_locations = []if os.path.isabs(folder_name):diff --git a/shared/utils/self_refiner.py b/shared/utils/self_refiner.pyindex 88b880b..fc10779 100644--- a/shared/utils/self_refiner.py+++ b/shared/utils/self_refiner.py@@ -110,7 +110,15 @@ def ensure_refiner_list(plan_data):def add_refiner_rule(current_rules, range_val, steps_val):current_rules = ensure_refiner_list(current_rules)- new_start, new_end = int(range_val[0]), int(range_val[1])+ if isinstance(range_val, str):+ raw_range = range_val.strip().replace(",", "-").replace(":", "-")+ if "-" in raw_range:+ start_s, end_s = raw_range.split("-", 1)+ else:+ start_s = end_s = raw_range+ new_start, new_end = int(start_s.strip()), int(end_s.strip())+ else:+ new_start, new_end = int(range_val[0]), int(range_val[1])if new_start > new_end:new_start, new_end = new_end, new_start@@ -455,4 +463,4 @@ def run_refinement_loop_multi(if _should_stop():break- return refined_latents_list\ No newline at end of file+ return refined_latents_listdiff --git a/wgp.py b/wgp.pyindex 4363440..3f57b11 100644--- a/wgp.py+++ b/wgp.py@@ -69,7 +69,10 @@ import shutilimport globimport cv2import html-from gradio_rangeslider import RangeSlider+try:+ from gradio_rangeslider import RangeSlider+except ImportError:+ RangeSlider = Noneimport refrom transformers.utils import logginglogging.set_verbosity_error@@ -460,44 +463,47 @@ def process_prompt_and_add_tasks(state, current_gallery_tab, model_choice):multi_prompts_gen_type = inputs["multi_prompts_gen_type"]if multi_prompts_gen_type in [0,2]:- if image_start != None and len(image_start) > 0:+ image_slots = image_start if image_start != None and len(image_start) > 0 else image_end+ if image_slots != None and len(image_slots) > 0:if inputs["multi_images_gen_type"] == 0:new_prompts = []new_image_start = []new_image_end = []- for i in range(len(prompts) * len(image_start) ):+ for i in range(len(prompts) * len(image_slots)):new_prompts.append( prompts[ i % len(prompts)] )- new_image_start.append(image_start[i // len(prompts)] )+ if image_start != None:+ new_image_start.append(image_start[i // len(prompts)] )if image_end != None:new_image_end.append(image_end[i // len(prompts)] )prompts = new_prompts- image_start = new_image_start- if image_end != None:- image_end = new_image_end+ image_start = new_image_start if image_start != None else None+ image_end = new_image_end if image_end != None else Noneelse:- if len(prompts) >= len(image_start):- if len(prompts) % len(image_start) != 0:+ if len(prompts) >= len(image_slots):+ if len(prompts) % len(image_slots) != 0:gr.Info("If there are more text prompts than input images the number of text prompts should be dividable by the number of images")return ret()- rep = len(prompts) // len(image_start)+ rep = len(prompts) // len(image_slots)new_image_start = []new_image_end = []for i, _ in enumerate(prompts):- new_image_start.append(image_start[i//rep] )+ if image_start != None:+ new_image_start.append(image_start[i//rep] )if image_end != None:new_image_end.append(image_end[i//rep] )- image_start = new_image_start- if image_end != None:- image_end = new_image_end+ image_start = new_image_start if image_start != None else None+ image_end = new_image_end if image_end != None else Noneelse:- if len(image_start) % len(prompts) !=0:+ if len(image_slots) % len(prompts) !=0:gr.Info("If there are more input images than text prompts the number of images should be dividable by the number of text prompts")return ret()- rep = len(image_start) // len(prompts)+ rep = len(image_slots) // len(prompts)new_prompts = []- for i, _ in enumerate(image_start):+ for i, _ in enumerate(image_slots):new_prompts.append( prompts[ i//rep] )prompts = new_prompts+ if image_start == None or len(image_start) == 0:+ image_start = [None] * len(prompts)if image_end == None or len(image_end) == 0:image_end = [None] * len(prompts)@@ -569,6 +575,17 @@ def get_model_custom_settings(model_def):normalized.append(one)return normalized++def end_frames_always_enabled(model_def):+ return bool((model_def or {}).get("end_frames_always_enabled", False))+++def end_frames_option_visible(model_def, image_prompt_type):+ if "E" not in (model_def or {}).get("image_prompt_types_allowed", ""):+ return False+ return end_frames_always_enabled(model_def) or any_letters(image_prompt_type or "", "SVL")++def parse_custom_setting_typed_value(raw_value, setting_type):if raw_value is None:return None, None@@ -987,7 +1004,7 @@ def validate_settings(state, model_type, single_prompt, inputs):else:image_start = None- if not any_letters(image_prompt_type, "SVL"):+ if not end_frames_always_enabled(model_def) and not any_letters(image_prompt_type, "SVL"):image_prompt_type = image_prompt_type.replace("E", "")if "E" in image_prompt_type:if image_end == None or isinstance(image_end, list) and len(image_end) == 0:@@ -1002,7 +1019,7 @@ def validate_settings(state, model_type, single_prompt, inputs):gr.Info("If you want to Continue a Video, you can use Multiple End Images only if the option 'Each Line Will be used for a new Sliding Window of the same Video Generation' is set")return ret()elif multi_prompts_gen_type in [0, 2]:- if len(image_start or []) != len(image_end or []):+ if len(image_start or []) > 0 and len(image_start or []) != len(image_end or []):gr.Info("The number of Start and End Images should be the same if the option 'Each Line Will be used for a new Sliding Window of the same Video Generation' is not set")return ret()else:@@ -2341,7 +2358,7 @@ if not Path(config_load_filename).is_file():"compile" : "","metadata_type": "metadata","boost" : 1,- "enable_int8_kernels": 0,+ "enable_int8_kernels": 1,"clear_file_list" : 5,"enable_4k_resolutions": 0,"max_reserved_loras": -1,@@ -2360,6 +2377,7 @@ if not Path(config_load_filename).is_file():"mmaudio_mode": 0,"mmaudio_persistence": 1,"rife_version": "v4",+ "prompt_enhancer_quantization": "quanto_int8","prompt_enhancer_temperature": 0.6,"prompt_enhancer_top_p": 0.9,"prompt_enhancer_randomize_seed": True,@@ -2373,6 +2391,8 @@ else:text = reader.read()server_config = json.loads(text)+server_config.setdefault("prompt_enhancer_quantization", "quanto_int8")+checkpoints_paths = server_config.get("checkpoints_paths", None)if checkpoints_paths is None: checkpoints_paths = server_config["checkpoints_paths"] = fl.default_checkpoints_pathsfl.set_checkpoints_paths(checkpoints_paths)@@ -2440,10 +2460,10 @@ _normalize_profile_defaults(server_config)_normalize_output_paths(server_config)lm_decoder_engine = server_config.get("lm_decoder_engine", "")-from preprocessing.matanyone.utils.model_signature import delete_if_not_matanyone2_model-mask_model_path = fl.locate_file("mask/model.safetensors", error_if_none=False)-if delete_if_not_matanyone2_model(mask_model_path):- print("Removing old version of 'mask/model.safetensors'. MatAnyone2 will be downloaded next time you use it.")+from preprocessing.matanyone.utils.model_assets import migrate_matanyone_install, query_matanyone_download_def+migration_note = migrate_matanyone_install(server_config)+if migration_note:+ print(migration_note)# Deprecated modelsfor path in ["wan2.1_Vace_1.3B_preview_bf16.safetensors", "sky_reels2_diffusion_forcing_1.3B_bf16.safetensors","sky_reels2_diffusion_forcing_720p_14B_bf16.safetensors",@@ -2982,7 +3002,7 @@ default_profile = default_profile_videoloaded_profile = force_profile_no = -1compile = server_config.get("compile", "")boost = server_config.get("boost", 1)-enable_int8_kernels = server_config.get("enable_int8_kernels", 0)+enable_int8_kernels = server_config.get("enable_int8_kernels", 1)apply_int8_kernel_setting(enable_int8_kernels)vae_config = server_config.get("vae_config", 0)if len(args.vae_config) > 0:@@ -3181,23 +3201,25 @@ def get_local_model_filename(model_filename, use_locator = True, extra_paths = Ndef process_files_def(repoId = None, sourceFolderList = None, fileList = None, targetFolderList = None):- original_targetRoot = fl.get_download_location()if targetFolderList is None:targetFolderList = [None] * len(sourceFolderList)for targetFolder, sourceFolder, files in zip(targetFolderList, sourceFolderList,fileList ):if targetFolder is not None and len(targetFolder) == 0: targetFolder = None- targetRoot = os.path.join(original_targetRoot, targetFolder) if targetFolder is not None else original_targetRoot+ explicit_target = targetFolder if targetFolder is not None else (sourceFolder if len(sourceFolder) > 0 else None)+ targetRoot = fl.get_smart_download_root(explicit_target)+ local_dir = os.path.join(targetRoot, targetFolder) if targetFolder is not None else targetRootif len(files)==0:if fl.locate_folder(sourceFolder if targetFolder is None else os.path.join(targetFolder, sourceFolder), error_if_none= False ) is None:- snapshot_download(repo_id=repoId, allow_patterns=sourceFolder +"/*", local_dir= targetRoot)+ snapshot_download(repo_id=repoId, allow_patterns=sourceFolder +"/*", local_dir= local_dir)else:for onefile in files:if len(sourceFolder) > 0:if fl.locate_file( (sourceFolder + "/" + onefile) if targetFolder is None else os.path.join(targetFolder, sourceFolder, onefile), error_if_none= False) is None:- hf_hub_download(repo_id=repoId, filename=onefile, local_dir = targetRoot, subfolder=sourceFolder)+ hf_hub_download(repo_id=repoId, filename=onefile, local_dir = local_dir, subfolder=sourceFolder)else:if fl.locate_file(onefile if targetFolder is None else os.path.join(targetFolder, onefile), error_if_none= False) is None:- hf_hub_download(repo_id=repoId, filename=onefile, local_dir = targetRoot)+ hf_hub_download(repo_id=repoId, filename=onefile, local_dir = local_dir)+def download_mmaudio():mmaudio_enabled, mmaudio_mode, _, _, _ = get_mmaudio_settings(server_config)@@ -3250,32 +3272,27 @@ def download_models(model_filename = None, model_type= None, file_type = 0, submif file_type == 0:shared_def = {"repoId" : "DeepBeepMeep/Wan2.1",- "sourceFolderList" : [ "pose", "scribble", "flow", "depth", "mask", "wav2vec", "chinese-wav2vec2-base", "roformer", "pyannote", "det_align", "" ],+ "sourceFolderList" : [ "pose", "scribble", "flow", "depth", "wav2vec", "chinese-wav2vec2-base", "roformer", "pyannote", "det_align", "" ],"fileList" : [ ["dw-ll_ucoco_384.onnx", "yolox_l.onnx"],["netG_A_latest.pth"], ["raft-things.pth"],- ["depth_anything_v2_vitl.pth","depth_anything_v2_vitb.pth"], ["sam_vit_h_4b8939_fp16.safetensors", "model.safetensors", "config.json"],+ ["depth_anything_v2_vitl.pth","depth_anything_v2_vitb.pth"],["config.json", "feature_extractor_config.json", "model.safetensors", "preprocessor_config.json", "special_tokens_map.json", "tokenizer_config.json", "vocab.json"],["config.json", "pytorch_model.bin", "preprocessor_config.json"],["model_bs_roformer_ep_317_sdr_12.9755.ckpt", "model_bs_roformer_ep_317_sdr_12.9755.yaml", "download_checks.json"],["pyannote_model_wespeaker-voxceleb-resnet34-LM.bin", "pytorch_model_segmentation-3.0.bin"], ["detface.pt"], [ RIFE_V3_FILENAME if server_config.get("rife_version", "v3") == "v3" else RIFE_V4_FILENAME ] ]}process_files_def(**shared_def)+ process_files_def(**query_matanyone_download_def(server_config))- if server_config.get("enhancer_enabled", 0) == 1:- enhancer_def = {- "repoId" : "DeepBeepMeep/LTX_Video",- "sourceFolderList" : [ "Florence2", "Llama3_2" ],- "fileList" : [ ["config.json", "configuration_florence2.py", "model.safetensors", "preprocessor_config.json", "tokenizer.json", "tokenizer_config.json"],["config.json", "generation_config.json", "Llama3_2_quanto_bf16_int8.safetensors", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json"] ]- }- process_files_def(**enhancer_def)+ enhancer_enabled = int(server_config.get("enhancer_enabled", 0) or 0)+ if enhancer_enabled > 0:+ from shared.prompt_enhancer import ensure_prompt_enhancer_assets- elif server_config.get("enhancer_enabled", 0) == 2:- enhancer_def = {- "repoId" : "DeepBeepMeep/LTX_Video",- "sourceFolderList" : [ "Florence2", "llama-joycaption-beta-one-hf-llava" ],- "fileList" : [ ["config.json", "configuration_florence2.py", "model.safetensors", "preprocessor_config.json", "tokenizer.json", "tokenizer_config.json"],["config.json", "llama_config.json", "llama_joycaption_quanto_bf16_int8.safetensors", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json"] ]- }- process_files_def(**enhancer_def)+ ensure_prompt_enhancer_assets(+ process_files_def,+ enhancer_enabled=enhancer_enabled,+ qwen_backend=server_config.get("prompt_enhancer_quantization", "quanto_int8"),+ )download_mmaudio()global download_shared_done@@ -3293,7 +3310,7 @@ def download_models(model_filename = None, model_type= None, file_type = 0, submif not (any_source and file_type==0 or any_module_source and file_type==1):local_model_filename = get_local_model_filename(model_filename, extra_paths= force_path)if local_model_filename is None and len(model_filename) > 0:- local_model_filename = fl.get_download_location(os.path.basename(model_filename), force_path= force_path)+ local_model_filename = fl.get_smart_download_location(os.path.basename(model_filename), force_path= force_path)url = model_filenameif not url.startswith("http"):@@ -3495,6 +3512,8 @@ def init_pipe(pipe, kwargs, profile):preload = server_config.get("preload_in_VRAM", 0)kwargs["extraModelsToQuantize"]= None+ source_budgets = kwargs.get("budgets", None)+ if source_budgets is None: kwargs["budgets"] = source_budgets = {}if profile in (2, 4, 5):default_transformer_budget = default_transformer2_budget= kwargs.get("budgets", 100)if isinstance(default_transformer_budget, dict):@@ -3504,9 +3523,9 @@ def init_pipe(pipe, kwargs, profile):budgets = { "transformer" : default_transformer_budget if preload == 0 else preload, "text_encoder" : 100 if preload == 0 else preload, "*" : max(1000 if profile==5 else 3000 , preload) }if "transformer2" in pipe:budgets["transformer2"] = default_transformer2_budget if preload == 0 else preload- kwargs["budgets"] = budgets+ source_budgets.update(budgets)elif profile == 3:- kwargs["budgets"] = { "*" : "70%" }+ source_budgets.update({ "*" : "70%" })if "transformer2" in pipe:if profile in [3,4]:@@ -3524,6 +3543,12 @@ def init_pipe(pipe, kwargs, profile):return mmgp_profilereset_prompt_enhancer_requested = False+def unload_prompt_enhancer_runtime():+ from shared.prompt_enhancer import unload_prompt_enhancer_models++ unload_prompt_enhancer_models(prompt_enhancer_image_caption_model, prompt_enhancer_llm_model)++def reset_prompt_enhancer():global reset_prompt_enhancer_requestedreset_prompt_enhancer_requested = True@@ -3533,6 +3558,7 @@ def reset_prompt_enhancer_if_requested():if not reset_prompt_enhancer_requested:returnreset_prompt_enhancer_requested = False+ unload_prompt_enhancer_runtime()prompt_enhancer_image_caption_model = Noneprompt_enhancer_image_caption_processor = Noneprompt_enhancer_llm_model = None@@ -3545,52 +3571,21 @@ def setup_prompt_enhancer(pipe, kwargs):global prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizermodel_no = server_config.get("enhancer_enabled", 0)if model_no != 0:- from shared.prompt_enhancer import load_florence2- from transformers import AutoTokenizer- prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor = load_florence2(- fl.locate_folder("Florence2"),- attn_implementation="sdpa",- )- pipe["prompt_enhancer_image_caption_model"] = prompt_enhancer_image_caption_model- prompt_enhancer_image_caption_model._model_dtype = torch.float- prompt_enhancer_image_caption_model.eval()-- if model_no == 1:- budget = 5000- prompt_enhancer_llm_model = offload.fast_load_transformers_model(- fl.locate_file("Llama3_2/Llama3_2_quanto_bf16_int8.safetensors"),- defaultConfigPath=fl.locate_file("Llama3_2/config.json", error_if_none=False),- configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},- writable_tensors=False,- )- prompt_enhancer_llm_model._validate_model_kwargs = lambda *_args, **_kwargs: None- prompt_enhancer_llm_model._offload_hooks = ["generate"]- prompt_enhancer_llm_tokenizer = AutoTokenizer.from_pretrained(fl.locate_folder("Llama3_2"))- else:- def preprocess_sd(sd, quant_map=None, tied_map=None):- rules = { "model.language_model": "model", "model.vision_tower": None, "model.multi_modal_projector": None, }- return tuple(offload.map_state_dict([sd, quant_map, tied_map], rules))-- budget = 10000- prompt_enhancer_llm_model = offload.fast_load_transformers_model(- fl.locate_file("llama-joycaption-beta-one-hf-llava/llama_joycaption_quanto_bf16_int8.safetensors"),- forcedConfigPath=fl.locate_file("llama-joycaption-beta-one-hf-llava/llama_config.json", error_if_none=False),- configKwargs={"attn_implementation": "sdpa", "hidden_act": "silu"},- preprocess_sd=preprocess_sd,- writable_tensors=False,- )+ from shared.prompt_enhancer import load_prompt_enhancer_runtime- prompt_enhancer_llm_tokenizer = AutoTokenizer.from_pretrained(fl.locate_folder("llama-joycaption-beta-one-hf-llava"))- prompt_enhancer_llm_model.generation_config.pad_token = prompt_enhancer_llm_tokenizer.eos_token- if prompt_enhancer_llm_model.generation_config.pad_token_id is None:- eos_token_id = prompt_enhancer_llm_model.generation_config.eos_token_id- prompt_enhancer_llm_model.generation_config.pad_token_id = eos_token_id [0] if isinstance(eos_token_id,list) else eos_token_id- prompt_enhancer_llm_model.eval()-- pipe["prompt_enhancer_llm_model"] = prompt_enhancer_llm_model-- if not "budgets" in kwargs: kwargs["budgets"] = {}- kwargs["budgets"]["prompt_enhancer_llm_model"] = budget+ runtime = load_prompt_enhancer_runtime(+ process_files_def,+ enhancer_enabled=model_no,+ lm_decoder_engine=server_config.get("lm_decoder_engine", ""),+ qwen_backend=server_config.get("prompt_enhancer_quantization", "quanto_int8"),+ )+ prompt_enhancer_image_caption_model = runtime.image_caption_model+ prompt_enhancer_image_caption_processor = runtime.image_caption_processor+ prompt_enhancer_llm_model = runtime.llm_model+ prompt_enhancer_llm_tokenizer = runtime.llm_tokenizer+ pipe.update(runtime.pipe_models)+ if runtime.budgets:+ kwargs.setdefault("budgets", {}).update(runtime.budgets)else:reset_prompt_enhancer()@@ -5418,6 +5413,7 @@ class DynamicClass:return self.assign(**dict)def process_prompt_enhancer(model_def, prompt_enhancer, original_prompts, image_start, original_image_refs, is_image, audio_only, seed, prompt_enhancer_instructions = None ):+ global enhancer_offloadobjprompt_enhancer_mode = str(prompt_enhancer or "")prompt_enhancer_instructions = model_def.get("image_prompt_enhancer_instructions" if is_image else "video_prompt_enhancer_instructions", None)text_encoder_max_tokens = model_def.get("image_prompt_enhancer_max_tokens" if is_image else "video_prompt_enhancer_max_tokens", 256)@@ -5436,7 +5432,7 @@ def process_prompt_enhancer(model_def, prompt_enhancer, original_prompts, imageif "I" in prompt_enhancer_mode:if image_start != None:if not isinstance(image_start, list): image_start= [image_start]- prompt_images += image_start+ prompt_images += image_start[:1]if original_image_refs != None:prompt_images += original_image_refs[:1]prompt_images = [Image.open(img) if isinstance(img,str) else img for img in prompt_images]@@ -5451,6 +5447,10 @@ def process_prompt_enhancer(model_def, prompt_enhancer, original_prompts, imageenhancer_seed = secrets.randbits(32)else:enhancer_seed = seed if seed is not None and seed >= 0 else 0+ post_image_caption_hook = None+ if len(prompt_images) > 0 and enhancer_offloadobj is not None:+ if hasattr(prompt_enhancer_image_caption_model, "vision_tower_model") and hasattr(prompt_enhancer_llm_model, "generate_messages"):+ post_image_caption_hook = enhancer_offloadobj.unload_allprompts = generate_cinematic_prompt(prompt_enhancer_image_caption_model,prompt_enhancer_image_caption_processor,@@ -5466,6 +5466,7 @@ def process_prompt_enhancer(model_def, prompt_enhancer, original_prompts, imagetemperature = enhancer_temperature,top_p = enhancer_top_p,seed = enhancer_seed,+ post_image_caption_hook = post_image_caption_hook,)return prompts@@ -5501,7 +5502,11 @@ def enhance_prompt(state, prompt, prompt_enhancer, multi_images_gen_type, multi_prompts_to_process= ["\n".join(prompts_to_process)]original_prompts = prompts_to_processnum_prompts = len(original_prompts)- image_start = inputs["image_start"]+ image_prompt_type = inputs["image_prompt_type"]+ video_prompt_type = inputs["video_prompt_type"]+ image_start = inputs["image_start"] if "S" in image_prompt_type else None+ if image_start is None:+ image_start = inputs["image_end"] if "E" in image_prompt_type else Noneif image_start is None or not "I" in prompt_enhancer:image_start = [None] * num_promptselse:@@ -5530,12 +5535,12 @@ def enhance_prompt(state, prompt, prompt_enhancer, multi_images_gen_type, multi_acquire_GPU_ressources(state, "prompt_enhancer", "Prompt Enhancer")if enhancer_offloadobj is None:+ setup_prompt_enhancer(pipe, kwargs)profile = compute_profile(override_profile, "video")mmgp_profile = init_pipe(pipe, kwargs, profile)- setup_prompt_enhancer(pipe, kwargs)enhancer_offloadobj = offload.profile(pipe, profile_no= mmgp_profile, **kwargs)- original_image_refs = inputs["image_refs"]+ original_image_refs = inputs["image_refs"] if "I" in video_prompt_type else Noneif original_image_refs is not None:original_image_refs = [ convert_image(tup[0]) for tup in original_image_refs ]is_image = inputs["image_mode"] > 0@@ -5550,6 +5555,7 @@ def enhance_prompt(state, prompt, prompt_enhancer, multi_images_gen_type, multi_try:enhanced_prompt = process_prompt_enhancer(model_def, prompt_enhancer, [one_prompt], start_images, original_image_refs, is_image, audio_only, seed)except Exception as e:+ unload_prompt_enhancer_runtime()enhancer_offloadobj.unload_all()release_GPU_ressources(state, "prompt_enhancer")raise gr.Error(e)@@ -5561,6 +5567,7 @@ def enhance_prompt(state, prompt, prompt_enhancer, multi_images_gen_type, multi_enhanced_prompts.append(prefix + " " + one_prompt)enhanced_prompts.append(enhanced_prompt)+ unload_prompt_enhancer_runtime()enhancer_offloadobj.unload_all()release_GPU_ressources(state, "prompt_enhancer")@@ -5637,6 +5644,28 @@ def slice_audio_window(audio_path, start_frame, num_frames, fps, output_dir, sufreturn data, sample_rate+def get_audio_file_sample_rate(audio_path):+ import ffmpeg++ probe = ffmpeg.probe(os.fspath(audio_path))+ audio_stream = next((stream for stream in probe["streams"] if stream.get("codec_type") == "audio"), None)+ if audio_stream is None or not audio_stream.get("sample_rate"):+ raise ValueError(f"Unable to read audio sample rate from {audio_path}")+ return int(audio_stream["sample_rate"])+++def resolve_mux_audio_sampling_rate(default_rate, source_audio_metadata=None, audio_paths=None):+ sample_rates = [int(default_rate)]+ for meta in source_audio_metadata or []:+ sample_rate = int(meta.get("sample_rate", 0) or 0)+ if sample_rate > 0:+ sample_rates.append(sample_rate)+ for audio_path in audio_paths or []:+ if audio_path:+ sample_rates.append(get_audio_file_sample_rate(audio_path))+ return max(sample_rates)++def custom_preprocess_video_with_mask(model_handler, base_model_type, pre_video_guide, video_guide, video_mask, height, width, max_frames, start_frame, fit_canvas, fit_crop, target_fps, block_size, expand_scale, video_prompt_type):pad_frames = 0if start_frame < 0:@@ -6035,7 +6064,7 @@ def generate_video(fps = 1 if is_image else get_computed_fps(force_fps, base_model_type , video_guide, video_source )control_audio_tracks = source_audio_tracks = source_audio_metadata = []if any_letters(audio_prompt_type, "R") and video_guide is not None and MMAudio_setting == 0 and not any_letters(audio_prompt_type, "ABXK"):- control_audio_tracks, _ = extract_audio_tracks(video_guide)+ control_audio_tracks, _ = extract_audio_tracks(video_guide, temp_format="wav")if "K" in audio_prompt_type and video_guide is not None:try:audio_guide = extract_audio_track_to_wav(video_guide, get_available_filename(save_path, video_guide, suffix="_control_audio", force_extension=".wav"))@@ -6045,7 +6074,7 @@ def generate_video(audio_guide = Noneaudio_guide2 = Noneif video_source is not None:- source_audio_tracks, source_audio_metadata = extract_audio_tracks(video_source)+ source_audio_tracks, source_audio_metadata = extract_audio_tracks(video_source, temp_format="wav")video_fps, _, _, video_frames_count = get_video_info(video_source)video_source_duration = video_frames_count / video_fpselse:@@ -6240,7 +6269,8 @@ def generate_video(start_time = time.time()if prompt_enhancer_image_caption_model != None and prompt_enhancer !=None and len(prompt_enhancer)>0 and enhancer_mode == 0:send_cmd("progress", [0, get_latest_status(state, "Enhancing Prompt")])- enhanced_prompts = process_prompt_enhancer(model_def, prompt_enhancer, original_prompts, image_start, original_image_refs, is_image, audio_only, seed )+ enhanced_prompts = process_prompt_enhancer(model_def, prompt_enhancer, original_prompts, image_start if image_start is not None else image_end , original_image_refs, is_image, audio_only, seed )+ unload_prompt_enhancer_runtime()if enhanced_prompts is not None:print(f"Enhanced prompts: {enhanced_prompts}" )task["prompt"] = "\n".join(["!enhanced!"] + enhanced_prompts)@@ -6306,6 +6336,9 @@ def generate_video(# image_end_tensor =image_end_list[window_no-1].resize((new_width, new_height), resample=Image.Resampling.LANCZOS)refresh_preview["image_end"] = image_end_tensorimage_end_tensor = convert_image_to_tensor(image_end_tensor)+ if sample_fit_canvas != None:+ image_size = image_end_tensor.shape[-2:]+ sample_fit_canvas = Noneimage_end_list= Nonewindow_start_frame = guide_start_frame - (reuse_frames if window_no > 1 else source_video_overlap_frames_count)guide_end_frame = guide_start_frame + current_video_length - (source_video_overlap_frames_count if window_no == 1 else reuse_frames)@@ -6901,16 +6934,18 @@ def generate_video(else:new_audio_tracks = control_audio_tracksif generated_audio is not None: output_new_audio_filepath = None-+ mux_audio_sampling_rate = resolve_mux_audio_sampling_rate(output_audio_sampling_rate, source_audio_metadata, new_audio_tracks)+combine_and_concatenate_video_with_audio_tracks(video_path,save_path_tmp,source_audio_tracks,new_audio_tracks,source_audio_duration,- output_audio_sampling_rate,+ mux_audio_sampling_rate,new_audio_from_start=new_audio_added_from_audio_start,source_audio_metadata=source_audio_metadata,+ audio_codec_key=server_config.get("audio_output_codec", "aac_128"),verbose=verbose_level >= 2,)os.remove(save_path_tmp)@@ -8982,8 +9017,7 @@ def refresh_image_prompt_type_radio(state, image_prompt_type, image_prompt_type_image_prompt_type = add_to_sequence(image_prompt_type, image_prompt_type_radio)any_video_source = len(filter_letters(image_prompt_type, "VL"))>0model_def = get_model_def(get_state_model_type(state))- image_prompt_types_allowed = model_def.get("image_prompt_types_allowed", "")- end_visible = "E" in image_prompt_types_allowed and any_letters(image_prompt_type, "SVL")+ end_visible = end_frames_option_visible(model_def, image_prompt_type)input_strength_visible = len(model_def.get("input_video_strength","")) and any_letters(image_prompt_type, "SVL")return image_prompt_type, gr.update(visible = "S" in image_prompt_type ), gr.update(visible = end_visible and ("E" in image_prompt_type) ), gr.update(visible = "V" in image_prompt_type) , gr.update(visible = input_strength_visible), gr.update(visible = any_video_source), gr.update(visible = end_visible)@@ -8991,7 +9025,8 @@ def refresh_image_prompt_type_endcheckbox(state, image_prompt_type, image_promptimage_prompt_type = del_in_sequence(image_prompt_type, "E")if end_checkbox: image_prompt_type += "E"image_prompt_type = add_to_sequence(image_prompt_type, image_prompt_type_radio)- return image_prompt_type, gr.update(visible = "E" in image_prompt_type )+ model_def = get_model_def(get_state_model_type(state))+ return image_prompt_type, gr.update(visible = end_frames_option_visible(model_def, image_prompt_type) and "E" in image_prompt_type)def refresh_video_prompt_type_image_refs(state, video_prompt_type, video_prompt_type_image_refs, image_mode):model_type = get_state_model_type(state)@@ -9672,6 +9707,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non# Video Continue / Start Frame / End Frameimage_prompt_type_value= ui_get("image_prompt_type")image_prompt_type = gr.Text(value= image_prompt_type_value, visible= False)+ end_option_visible = end_frames_option_visible(model_def, image_prompt_type_value) and not image_outputsimage_prompt_type_choices = []if "T" in image_prompt_types_allowed:image_prompt_type_choices += [("Text Prompt Only" if "S" in image_prompt_types_allowed else "New Video", "")]@@ -9693,7 +9729,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonelse:image_prompt_type_radio = gr.Radio(choices=[("", "")], value="", visible= False)if "E" in image_prompt_types_allowed:- image_prompt_type_endcheckbox = gr.Checkbox( value ="E" in image_prompt_type_value, label="End Image(s)", show_label= False, visible= any_letters(image_prompt_type_radio_value, "SVL") and not image_outputs , scale= 1, elem_classes="cbx_centered")+ image_prompt_type_endcheckbox = gr.Checkbox( value ="E" in image_prompt_type_value, label="End Image(s)", show_label= False, visible=end_option_visible, scale= 1, elem_classes="cbx_centered")any_end_image = Trueelse:image_prompt_type_endcheckbox = gr.Checkbox( value =False, show_label= False, visible= False , scale= 1)@@ -9701,7 +9737,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonvideo_source = gr.Video(label= "Video to Continue", height = gallery_height, visible= "V" in image_prompt_type_value, value= ui_defaults.get("video_source", None), elem_id="video_input")input_video_strength_label = model_def.get("input_video_strength", "")input_video_strength = gr.Slider(0, 1, value=ui_get("input_video_strength", 1.0), step=0.01, label=input_video_strength_label, visible = len(input_video_strength_label) and any_letters(image_prompt_type_value, "SVL"), show_reset_button= False)- image_end_row, image_end, image_end_extra = get_image_gallery(label= get_image_end_label(ui_get("multi_prompts_gen_type")), value = ui_defaults.get("image_end", None), visible= any_letters(image_prompt_type_value, "SVL") and ("E" in image_prompt_type_value) )+ image_end_row, image_end, image_end_extra = get_image_gallery(label= get_image_end_label(ui_get("multi_prompts_gen_type")), value = ui_defaults.get("image_end", None), visible=end_option_visible and "E" in image_prompt_type_value)if model_mode_choices is None:model_mode = gr.Dropdown(value=None, label="model mode", visible=False)else:@@ -10562,7 +10598,10 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nongr.Markdown("#### Refiner Rules")with gr.Row(elem_id="refiner-input-row"):- refiner_range = RangeSlider(minimum=1, maximum=100, value=(1, 10), step=1, label="Step Range", info="Start - End", scale=3)+ if RangeSlider is not None:+ refiner_range = RangeSlider(minimum=1, maximum=100, value=(1, 10), step=1, label="Step Range", info="Start - End", scale=3)+ else:+ refiner_range = gr.Textbox(value="1-10", label="Step Range", info="Start-End", scale=3)refiner_mult = gr.Slider(label="Iterations", value=3, minimum=1, maximum=5, step=1, scale=2)refiner_add_btn = gr.Button("➕ Add", variant="primary", scale=0, min_width=100)