FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Qwen VL vision tower: full-attention ViT blocks and a 2x2 patch merger; DeepStack taps ride along as extra output columns."""
from __future__ import annotations
import os
from typing import TYPE_CHECKING, List
import torch
import torch.nn.functional as F
from freetoken.distributed import get_tp_info
from freetoken.layers import (
BaseOP,
LinearColParallelMerged,
LinearOProj,
LinearQKVMerged,
LinearRowParallel,
OPList,
)
from freetoken.utils import div_even
from freetoken.models.weight_stream import BlockWeightStreamer
if TYPE_CHECKING:
from freetoken.layers.quantization import QuantConfig
from freetoken.message import MMItem
from .config import VisionConfig
class VisionLayerNorm(BaseOP):
def __init__(self, size: int, eps: float = 1e-6):
self.weight = torch.empty(size)
self.bias = torch.empty(size)
self._size = size
self._eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
return F.layer_norm(x, (self._size,), self.weight, self.bias, self._eps)
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
half = x.shape[-1] // 2
return torch.cat((-x[..., half:], x[..., :half]), dim=-1)
_warned_rope_fallback = False
def _apply_vision_rope(
q: torch.Tensor, k: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, head_dim: int
) -> None:
"""In-place 2D rope through the NeoX text kernel: the [c,c]/[s,s] layout is the half-rotation layout and positions index per-token cache rows."""
global _warned_rope_fallback
S = q.shape[0]
try:
from freetoken.kernel.triton.rope import apply_rope_with_cos_sin_cache_inplace
apply_rope_with_cos_sin_cache_inplace(
positions, q.view(S, -1), k.view(S, -1), head_dim, cache, is_neox=True
)
return
except ImportError:
# only a missing triton falls back; a CUDA error must not re-run eager on a half-rotated q/k
if not _warned_rope_fallback:
_warned_rope_fallback = True
from freetoken.utils import init_logger
init_logger(__name__).warning("vision rope triton kernel unavailable; using eager")
half = cache.shape[1] // 2
cos = torch.cat((cache[:, :half], cache[:, :half]), dim=-1).unsqueeze(-2)
sin = torch.cat((cache[:, half:], cache[:, half:]), dim=-1).unsqueeze(-2)
q.copy_((q.float() * cos + _rotate_half(q.float()) * sin).to(q.dtype))
k.copy_((k.float() * cos + _rotate_half(k.float()) * sin).to(k.dtype))
class VisionAttention(BaseOP):
def __init__(self, vc: VisionConfig, *, quant_config: QuantConfig | None = None, prefix: str = ""):
self.num_heads = div_even(vc.num_heads, get_tp_info().size)
self.head_dim = vc.hidden_size // vc.num_heads
self.qkv = LinearQKVMerged(
vc.hidden_size, self.head_dim, vc.num_heads, vc.num_heads, has_bias=True,
quant_config=quant_config, prefix=f"{prefix}.qkv",
)
self.proj = LinearOProj(
vc.hidden_size, vc.hidden_size, has_bias=True, quant_config=quant_config, prefix=f"{prefix}.proj"
)
def attend(
self, qkv: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, lengths: List[int]
) -> torch.Tensor:
"""Rope + bidirectional attention within each image on the fused qkv projection; returns [S, heads * head_dim]."""
S = qkv.shape[0]
# q/k/v stay views into the fused projection: the rope kernel takes strides, SDPA needs only head_dim contiguous
q, k, v = qkv.view(S, 3, self.num_heads, self.head_dim).unbind(1)
_apply_vision_rope(q, k, cache, positions, self.head_dim)
q = q.transpose(0, 1).unsqueeze(0)
k = k.transpose(0, 1).unsqueeze(0)
v = v.transpose(0, 1).unsqueeze(0)
# bidirectional attention within each image; images never attend across
outs = [
F.scaled_dot_product_attention(qs, ks, vs)
for qs, ks, vs in zip(
torch.split(q, lengths, dim=2),
torch.split(k, lengths, dim=2),
torch.split(v, lengths, dim=2),
)
]
o = outs[0] if len(outs) == 1 else torch.cat(outs, dim=2)
return o[0].transpose(0, 1).reshape(S, -1)
class VisionMLP(BaseOP):
def __init__(self, vc: VisionConfig, *, quant_config: QuantConfig | None = None, prefix: str = ""):
self.linear_fc1 = LinearColParallelMerged(
vc.hidden_size, [vc.intermediate_size], has_bias=True, quant_config=quant_config, prefix=f"{prefix}.linear_fc1"
)
self.linear_fc2 = LinearRowParallel(
vc.intermediate_size, vc.hidden_size, has_bias=True, quant_config=quant_config, prefix=f"{prefix}.linear_fc2"
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if x.shape[0] <= _MLP_ROWS:
return self.linear_fc2.forward(F.gelu(self.linear_fc1.forward(x), approximate="tanh"))
# per-token op: row chunks bound the two [rows, intermediate] temporaries while the GEMMs stay large
out = torch.empty_like(x)
for lo in range(0, x.shape[0], _MLP_ROWS):
rows = x[lo : lo + _MLP_ROWS]
out[lo : lo + _MLP_ROWS] = self.linear_fc2.forward(F.gelu(self.linear_fc1.forward(rows), approximate="tanh"))
return out
# 4096 rows x 4304 x bf16 = 34 MiB per temporary while the GEMM still fills the GPU
# FREETOKEN_VIT_MLP_ROWS=0 disables the chunking; cuBLAS may round a row chunk one bf16 ulp differently from the whole image
_MLP_ROWS = int(os.environ.get("FREETOKEN_VIT_MLP_ROWS", 4096)) or 1 << 62
# 16384 rows x 1152 x fp32 = 72 MiB per interpolation temporary
_POS_ROWS = 16384
class VisionBlock(BaseOP):
def __init__(self, vc: VisionConfig, *, quant_config: QuantConfig | None = None, prefix: str = ""):
self.norm1 = VisionLayerNorm(vc.hidden_size)
self.norm2 = VisionLayerNorm(vc.hidden_size)
self.attn = VisionAttention(vc, quant_config=quant_config, prefix=f"{prefix}.attn")
self.mlp = VisionMLP(vc, quant_config=quant_config, prefix=f"{prefix}.mlp")
def forward(
self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, lengths: List[int]
) -> torch.Tensor:
x = x + self._attention(x, cache, positions, lengths)
return x + self.mlp.forward(self.norm2.forward(x))
def _attention(
self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, lengths: List[int]
) -> torch.Tensor:
# each temporary dies inside the next call: norm out in the qkv GEMM, qkv in attention, its output in proj
o = self.attn.attend(self.attn.qkv.forward(self.norm1.forward(x)), cache, positions, lengths)
return self.attn.proj.forward(o)
class _Conv3dParams(BaseOP):
def __init__(self, out_ch: int, shape: tuple):
self.weight = torch.empty(out_ch, *shape)
self.bias = torch.empty(out_ch)
forward = None
class _EmbeddingParams(BaseOP):
def __init__(self, rows: int, dim: int):
self.weight = torch.empty(rows, dim)
forward = None
class VisionConv3dPatchEmbed(BaseOP):
"""Kept as a real Conv3d (kernel == stride): an equivalent unfolded matmul rounds differently."""
def __init__(self, vc: VisionConfig):
self._shape = (vc.in_channels, vc.temporal_patch_size, vc.patch_size, vc.patch_size)
self.proj = _Conv3dParams(vc.hidden_size, self._shape)
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
x = pixel_values.view(-1, *self._shape).to(self.proj.weight.dtype)
out = F.conv3d(x, self.proj.weight, self.proj.bias, stride=self._shape[1:])
return out.view(x.shape[0], -1)
class VisionPatchMerger(BaseOP):
def __init__(
self,
vc: VisionConfig,
use_postshuffle_norm: bool = False,
*,
quant_config: QuantConfig | None = None,
prefix: str = "",
):
merged = vc.hidden_size * vc.spatial_merge_size**2
# the DeepStack mergers normalize the 2x2-merged vector, the final merger each patch
self.norm = VisionLayerNorm(merged if use_postshuffle_norm else vc.hidden_size)
self.linear_fc1 = LinearColParallelMerged(
merged, [merged], has_bias=True, quant_config=quant_config, prefix=f"{prefix}.linear_fc1"
)
self.linear_fc2 = LinearRowParallel(
merged, vc.out_hidden_size, has_bias=True, quant_config=quant_config, prefix=f"{prefix}.linear_fc2"
)
self._merged = merged
self._postshuffle_norm = use_postshuffle_norm
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self._postshuffle_norm:
x = self.norm.forward(x.view(-1, self._merged))
else:
x = self.norm.forward(x).view(-1, self._merged)
return self.linear_fc2.forward(F.gelu(self.linear_fc1.forward(x)))
class Qwen3VLVisionModel(BaseOP):
"""Pixels -> per-image soft tokens [sum(h*w)/merge^2, out_hidden x (1 + DeepStack levels)].
Columns past out_hidden are the DeepStack side features in deepstack_visual_indexes order.
"""
def __init__(self, vc: VisionConfig, *, quant_config: QuantConfig | None = None, prefix: str = "visual"):
self.patch_embed = VisionConv3dPatchEmbed(vc)
self.pos_embed = _EmbeddingParams(vc.num_position_embeddings, vc.hidden_size)
self.blocks = OPList(
[VisionBlock(vc, quant_config=quant_config, prefix=f"{prefix}.blocks.{i}") for i in range(vc.depth)]
)
self.merger = VisionPatchMerger(vc, quant_config=quant_config, prefix=f"{prefix}.merger")
self.deepstack_merger_list = OPList(
[
VisionPatchMerger(
vc, use_postshuffle_norm=True, quant_config=quant_config, prefix=f"{prefix}.deepstack_merger_list.{k}"
)
for k in range(len(vc.deepstack_visual_indexes))
]
)
self._vc = vc
self._num_grid_per_side = int(vc.num_position_embeddings**0.5)
self._inv_dim = (vc.hidden_size // vc.num_heads) // 2
self._inv_freq: torch.Tensor | None = None
self._streamer: BlockWeightStreamer | None = None
def place_weights(self, mode: str) -> None:
"""gpu: every tensor resident; host: block tensors in pinned banks streamed two blocks at a time (mergers and embeddings stay resident)."""
if mode == "host" and self._streamer is None:
self._streamer = BlockWeightStreamer(self.blocks.op_list, self.pos_embed.weight.device)
elif mode == "gpu" and self._streamer is not None:
self._streamer.unstream()
self._streamer = None
elif mode not in ("gpu", "host"):
raise ValueError(f"unknown vision weight placement {mode!r}")
def _blocks(self):
if self._streamer is None:
return enumerate(self.blocks.op_list)
return self._streamer.blocks(self.blocks.op_list)
def _add_pos_embed(self, x: torch.Tensor, grid: torch.Tensor) -> None:
"""x += bilinear resample of the position table, in row chunks so the fp32 temporaries stay small."""
from transformers.vision_utils import get_vision_interpolation_indices_and_weights
interp_idx, interp_w = get_vision_interpolation_indices_and_weights(
grid,
num_grid_per_side=self._num_grid_per_side,
mode="bilinear",
align_corners=True,
spatial_merge_size=self._vc.spatial_merge_size,
)
table = self.pos_embed.weight
for lo in range(0, x.shape[0], _POS_ROWS):
idx, w = interp_idx[lo : lo + _POS_ROWS], interp_w[lo : lo + _POS_ROWS]
# one corner at a time, multiply then add: the same per-element arithmetic as the fused [S, 4, hidden] .sum(1)
acc = F.embedding(idx[:, 0], table).float() * w[:, 0, None]
for k in range(1, idx.shape[1]):
acc += F.embedding(idx[:, k], table).float() * w[:, k, None]
x[lo : lo + _POS_ROWS].add_(acc.to(x.dtype))
def _rope_table(self, grid: torch.Tensor, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
from transformers.vision_utils import get_vision_position_ids
if self._inv_freq is None or self._inv_freq.device != device:
# built lazily: the model itself is constructed on the meta device
# kept in the model dtype (bf16) on purpose; fp32 here would change the rope numerics
inv_dim = self._inv_dim
self._inv_freq = (
1.0 / (10000.0 ** (torch.arange(0, inv_dim, 2, dtype=torch.float32, device=device) / inv_dim))
).to(self.pos_embed.weight.dtype)
pos_ids = get_vision_position_ids(grid, self._vc.spatial_merge_size)
# long * bf16 -> bf16 freqs on purpose; fp32 freqs would round differently
freqs = (pos_ids.unsqueeze(-1) * self._inv_freq).flatten(1) # [S, head_dim/2]
# per-token rope rows [cos | sin] consumed by the NeoX kernel via positions=arange
cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1).to(torch.float32).contiguous()
positions = torch.arange(pos_ids.shape[0], dtype=torch.int32, device=device)
return cache, positions
def _embed(self, pixel_values: torch.Tensor, grid_thw: List[List[int]]) -> tuple[torch.Tensor, tuple]:
"""Patch embeddings with the position table added, and the attention inputs every block takes."""
from transformers.vision_utils import get_vision_cu_seqlens
device = self.pos_embed.weight.device
grid = torch.tensor(grid_thw, dtype=torch.long, device=device)
x = self.patch_embed.forward(pixel_values.to(device))
self._add_pos_embed(x, grid)
cache, positions = self._rope_table(grid, device)
return x, (cache, positions, torch.diff(get_vision_cu_seqlens(grid)).tolist())
def _encode(
self, pixel_values: torch.Tensor, grid_thw: List[List[int]], groups: torch.Tensor | None = None
) -> torch.Tensor:
"""Pixels through the patch embedding and the block stack; with ``groups`` each DeepStack tap's merged feature is copied into groups[1 + k]."""
taps = self._vc.deepstack_visual_indexes
x, attn = self._embed(pixel_values, grid_thw)
for i, blk in self._blocks():
x = blk.forward(x, *attn)
if groups is not None and i in taps:
k = taps.index(i)
groups[1 + k].copy_(self.deepstack_merger_list.op_list[k].forward(x), non_blocking=True)
return x
@staticmethod
def _unpark(groups: torch.Tensor, device: torch.device) -> torch.Tensor:
"""[rows, n * width] on the device from host slabs [n, rows, width]; group k fills columns k*width:(k+1)*width."""
n, rows, width = groups.shape
out = torch.empty(rows, n * width, dtype=groups.dtype, device=device)
for k in range(n):
out[:, k * width : (k + 1) * width].copy_(groups[k], non_blocking=True)
return out
@torch.inference_mode()
def forward(self, pixel_values: torch.Tensor, grid_thw: List[List[int]]) -> torch.Tensor:
"""Soft tokens [rows, width x (1 + taps)], rows = sum(h*w) / merge^2: the merger's columns first, then one DeepStack tap per range.
Same numbers as ``forward_naive``.
"""
vc = self._vc
taps = vc.deepstack_visual_indexes
if not taps:
return self.merger.forward(self._encode(pixel_values, grid_thw))
# the merger outputs leave the GPU as they are produced and come back after the last block, so only the block working set is resident while the blocks run
rows = pixel_values.shape[0] // vc.spatial_merge_size**2
groups = torch.empty(
(1 + len(taps), rows, vc.out_hidden_size), dtype=self.pos_embed.weight.dtype, device="cpu",
pin_memory=torch.cuda.is_available(),
)
x = self._encode(pixel_values, grid_thw, groups)
groups[0].copy_(self.merger.forward(x), non_blocking=True)
del x
return self._unpark(groups, self.pos_embed.weight.device)
@torch.inference_mode()
def forward_naive(self, pixel_values: torch.Tensor, grid_thw: List[List[int]]) -> torch.Tensor:
"""Reference forward, written the plain way: the block stack, then the merger outputs side by side."""
taps = self._vc.deepstack_visual_indexes
x, attn = self._embed(pixel_values, grid_thw)
features = []
for i, blk in self._blocks():
x = blk.forward(x, *attn)
if i in taps:
features.append(self.deepstack_merger_list.op_list[taps.index(i)].forward(x))
return torch.cat([self.merger.forward(x)] + features, dim=1)
class QwenVLVisionMixin:
"""The engine hooks of a wrapper that owns a Qwen VL tower as ``self.visual``."""
visual: Qwen3VLVisionModel
def place_encoder_weights(self, mode: str) -> None:
self.visual.place_weights(mode)
def encode(self, item: MMItem) -> torch.Tensor:
return self.visual.forward(item.feature, [item.grid_thw])
__all__ = ["Qwen3VLVisionModel", "QwenVLVisionMixin"]