Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(rotary): size the rope table for YaRN and add --hf-overrides (#573)

YevheniiKotyrlo235e2016ddb6
diff --git a/docs/cli.md b/docs/cli.md
index 0c339439..fb5e613f 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -33,6 +33,7 @@ parsers all resolve automatically from the checkpoint and the GPU.
|---|---|---|
| `--model-path`, `--model` | required | Local dir, HF repo id, or an FTW dir (auto-detected) |
| `--served-model-name` | basename of `--model` | Model id reported by `/v1/models` |
+| `--hf-overrides` | — | JSON object applied to the checkpoint's config as vLLM's `--hf-overrides`: a nested config section updates key by key, any other value is replaced whole. A YaRN `rope_parameters` override serves `original_max_position_embeddings * factor` positions |
### Server & runtime
diff --git a/python/freetoken/attention/qsa_sparse.py b/python/freetoken/attention/qsa_sparse.py
index b07c9523..8ce63c42 100644
--- a/python/freetoken/attention/qsa_sparse.py
+++ b/python/freetoken/attention/qsa_sparse.py
@@ -167,23 +167,22 @@ class QSASparseAttnBackend(BaseAttnBackend):
def _index_rope_cache(self) -> torch.Tensor:
"""cos/sin table of the indexer rope: same rotary_dim and frequencies as the main
- attention, ``head_size`` 128 instead of 256, so it is a separate get_rope instance.
+ attention with ``head_size`` 128 instead of 256, so the attention's own table.
The table itself (not RotaryEmbedding.forward) because the indexer's norm+rope is one
fused kernel and the compressed keys rope at their group's position, not the query's."""
if self._index_cos_sin is None:
- from freetoken.layers.rotary import get_rope
+ from freetoken.layers.rotary import rope_cos_sin_table
rotary = self.rotary_config
with torch.device(self.device):
- rope = get_rope(
- head_dim=self.index_head_dim,
- rotary_dim=rotary.rotary_dim,
- max_position=rotary.max_position,
- base=rotary.base,
- rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,
+ self._index_cos_sin = rope_cos_sin_table(
+ self.index_head_dim,
+ rotary.rotary_dim,
+ rotary.max_position,
+ rotary.base,
+ tuple(rotary.scaling.items()) if rotary.scaling else None,
)
- self._index_cos_sin = rope._cos_sin_cache.to(self.device)
return self._index_cos_sin
def _index_rope_rows(self, positions: torch.Tensor) -> torch.Tensor:
diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.py
index 44c5a38e..3e38cb21 100644
--- a/python/freetoken/engine/config.py
+++ b/python/freetoken/engine/config.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import copy
from dataclasses import dataclass, field, replace
from functools import cached_property
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, Any, List, Mapping
import torch
from freetoken.distributed import DistributedInfo
@@ -23,6 +23,8 @@ class EngineConfig:
model_path: str
tp_info: DistributedInfo
dtype: torch.dtype
+ # --hf-overrides: applied to the checkpoint config the model is built from (cached_load_hf_config)
+ hf_overrides: Mapping[str, Any] = field(default_factory=dict)
max_running_req: int = 4
attention_backend: str = "auto"
moe_strategy: str = "auto"
@@ -103,7 +105,7 @@ class EngineConfig:
@cached_property
def hf_config(self):
- return cached_load_hf_config(self.model_path)
+ return cached_load_hf_config(self.model_path, self.hf_overrides)
@cached_property
def model_spec(self) -> ModelSpec:
@@ -142,7 +144,7 @@ class EngineConfig:
def max_seq_len(self) -> int:
if self.max_seq_len_override is not None:
return self.max_seq_len_override
- return self.model_config.rotary_config.max_position
+ return self.model_config.rotary_config.table_positions
@property
def max_forward_len(self) -> int:
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index cc14cf53..e855bcd3 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -1770,18 +1770,17 @@ def _adjust_config(config: EngineConfig):
)
override("num_page_override", config.num_token_override // config.page_size)
- # The rope cos/sin table is baked to rotary_config.max_position, and neither rope kernel
+ # The rope cos/sin table covers rotary_config.table_positions, and neither rope kernel
# bounds-checks the position it gathers with -- a longer ceiling reads past the table.
# DSV4 is exempt: it sizes its own table from the resolved max_seq_len (_adjust_dsv4_config).
rotary = getattr(model_config, "rotary_config", None)
seq_override = getattr(config, "max_seq_len_override", None)
if seq_override is not None and rotary is not None and not is_dsv4:
- if seq_override > rotary.max_position:
+ if seq_override > rotary.table_positions:
raise ValueError(
f"--max-seq-len-override {seq_override} exceeds the model's "
- f"rope table ({rotary.max_position} positions). Serving past it would read "
- "out of bounds; extend the checkpoint's rope_scaling / "
- "max_position_embeddings in config.json instead."
+ f"rope table ({rotary.table_positions} positions). Serving past it would read "
+ "out of bounds; extend the rope with --hf-overrides (YaRN rope_parameters) instead."
)
# The startup ServerArgs dump is the *requested* config, printed in the frontend process
diff --git a/python/freetoken/layers/rotary.py b/python/freetoken/layers/rotary.py
index 0d04645d..d0178f3b 100644
--- a/python/freetoken/layers/rotary.py
+++ b/python/freetoken/layers/rotary.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import functools
import math
-from typing import Any, Callable, Dict, Tuple
+from typing import Any, Mapping, Tuple
import torch
@@ -14,50 +14,21 @@ class RotaryEmbedding(StateLessOP):
self,
head_size: int,
rotary_dim: int,
- max_position_embeddings: int,
- base: float,
- post_process: None | Callable[[torch.Tensor], torch.Tensor] = None,
- proportional: bool = False,
- attention_factor: float = 1.0,
+ cos_sin_cache: torch.Tensor,
is_neox: bool = True,
) -> None:
super().__init__()
+ assert 0 < rotary_dim <= head_size
+ assert rotary_dim % 2 == 0
+ assert head_size in [64, 128, 256, 512]
self.head_size = head_size
self.rotary_dim = rotary_dim
# NeoX (half-rotation, HF default) vs GPT-J interleaved (adjacent pairs,
# ``rope_interleave`` models: GLM MLA lineage). Both underlying kernels
# accept the flag; the cos/sin cache layout is identical.
self.is_neox = is_neox
- if proportional:
- assert 0 < rotary_dim <= head_size
- assert rotary_dim % 2 == 0
- inv_freq = 1.0 / (
- base ** (torch.arange(0, head_size, 2, dtype=torch.float) / head_size)
- )
- if rotary_dim < head_size:
- inv_freq[rotary_dim // 2 :] = 0.0
- else:
- # Standard (NeoX) rope. Supports partial rotary (rotary_dim < head_size):
- # rope is applied to the first ``rotary_dim`` dims of each head, the rest pass
- # through. Frequencies are spaced over ``rotary_dim`` (matches HF default
- # partial rope, e.g. Qwen3.5 partial_rotary_factor, MiniMax-M2's
- # ``apply_rotary_pos_emb``). Full rope is rotary_dim == head_size and is
- # unaffected. ``head_size`` is passed to flashinfer separately so it rotates
- # only the first ``rotary_dim`` dims.
- assert 0 < rotary_dim <= head_size
- assert rotary_dim % 2 == 0
- inv_freq = 1.0 / (
- base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim)
- )
- if post_process is not None:
- inv_freq = post_process(inv_freq)
- t = torch.arange(max_position_embeddings, dtype=torch.float)
- freqs = torch.einsum("i,j -> ij", t, inv_freq)
- cos = freqs.cos() * attention_factor
- sin = freqs.sin() * attention_factor
- # buffer, so don't load/save
- self._cos_sin_cache = torch.cat((cos, sin), dim=-1)
- assert self.head_size in [64, 128, 256, 512]
+ # shared with every rope of the same frequencies (rope_cos_sin_table), so never written
+ self._cos_sin_cache = cos_sin_cache
from freetoken.kernel.backend import is_flashinfer_installed
@@ -160,6 +131,7 @@ class MRotaryEmbedding(RotaryEmbedding):
) -> None:
super().__init__(*args, **kwargs)
assert self.is_neox, "mrope is defined on the NeoX half-rotation layout"
+ assert self._cos_sin_cache.shape[1] == self.rotary_dim, "mrope reads a rotary_dim-wide table, not a proportional rope's"
half = self.rotary_dim // 2
assert sum(mrope_section) == half, (mrope_section, half)
self._section_table = build_section_table(tuple(mrope_section), layout)
@@ -193,56 +165,50 @@ class MRotaryEmbedding(RotaryEmbedding):
return query, key
-def _get_rope(
- head_dim: int,
- rotary_dim: int,
- max_position: int,
- base: float,
- rope_scaling: Dict[str, Any] | None = None,
- is_neox: bool = True,
-) -> RotaryEmbedding:
- if rope_scaling is None:
- return RotaryEmbedding(head_dim, rotary_dim, max_position, base, is_neox=is_neox)
- # need to test some cases:
- match rope_scaling["rope_type"]:
+def rope_table_positions(max_position: int, rope_scaling: Mapping[str, Any] | None) -> int:
+ """Rows the cos/sin table needs: YaRN serves ``original_max_position_embeddings * factor``,
+ above ``max_position_embeddings`` when only the rope parameters are overridden."""
+ if rope_scaling is None or rope_scaling.get("rope_type") != "yarn":
+ return max_position
+ extended = int(rope_scaling["original_max_position_embeddings"] * rope_scaling["factor"])
+ return max(max_position, extended)
+
+
+def _rope_frequencies(
+ head_size: int, rotary_dim: int, base: float, rope_scaling: Mapping[str, Any] | None
+) -> Tuple[torch.Tensor, float]:
+ """``inv_freq`` over the table's half width, and the factor cos and sin are both scaled by."""
+ rope_type = "default" if rope_scaling is None else rope_scaling["rope_type"]
+ if rope_type == "proportional":
+ # Frequencies spaced over the whole head; the pairs past rotary_dim do not rotate.
+ inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2, dtype=torch.float) / head_size))
+ inv_freq[rotary_dim // 2 :] = 0.0
+ return inv_freq, 1.0
+ # Standard (NeoX) rope. Supports partial rotary (rotary_dim < head_size): rope is applied
+ # to the first ``rotary_dim`` dims of each head, the rest pass through, and frequencies
+ # are spaced over ``rotary_dim`` (HF default partial rope, e.g. Qwen3.5
+ # partial_rotary_factor, MiniMax-M2's ``apply_rotary_pos_emb``).
+ inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
+ match rope_type:
case "default":
- return RotaryEmbedding(head_dim, rotary_dim, max_position, base, is_neox=is_neox)
-
- case "proportional":
- return RotaryEmbedding(
- head_dim,
- rotary_dim,
- max_position,
- base,
- proportional=True,
- is_neox=is_neox,
- )
+ return inv_freq, 1.0
case "llama3":
scaling_factor: float = rope_scaling["factor"]
low_freq_factor: float = rope_scaling["low_freq_factor"]
high_freq_factor: float = rope_scaling["high_freq_factor"]
original_max_position: int = rope_scaling["original_max_position_embeddings"]
-
- def post_process(inv_freq: torch.Tensor) -> torch.Tensor:
- # no smooth if low_freq_factor == high_freq_factor
- wave_len = 2 * math.pi / inv_freq
- if low_freq_factor == high_freq_factor:
- return torch.where(
- wave_len < original_max_position / high_freq_factor,
- inv_freq,
- inv_freq / scaling_factor,
- )
-
- delta = high_freq_factor - low_freq_factor
- smooth = (original_max_position / wave_len - low_freq_factor) / delta
- smooth = torch.clamp(smooth, 0, 1)
- factor = (1 - smooth) / scaling_factor + smooth
- return factor * inv_freq
-
- return RotaryEmbedding(
- head_dim, rotary_dim, max_position, base, post_process, is_neox=is_neox
- )
+ wave_len = 2 * math.pi / inv_freq
+ # no smooth if low_freq_factor == high_freq_factor
+ if low_freq_factor == high_freq_factor:
+ return torch.where(
+ wave_len < original_max_position / high_freq_factor,
+ inv_freq,
+ inv_freq / scaling_factor,
+ ), 1.0
+ delta = high_freq_factor - low_freq_factor
+ smooth = torch.clamp((original_max_position / wave_len - low_freq_factor) / delta, 0, 1)
+ return ((1 - smooth) / scaling_factor + smooth) * inv_freq, 1.0
case "yarn":
factor: float = rope_scaling["factor"]
@@ -288,27 +254,54 @@ def _get_rope(
high = min(high, rotary_dim - 1)
if low == high: # HF nudges instead of flooring the gap at 1 ("truncate": false)
high += 0.001
-
- def post_process(inv_freq: torch.Tensor) -> torch.Tensor:
- ramp = torch.clamp(
- (torch.arange(rotary_dim // 2, dtype=torch.float32) - low) / (high - low),
- 0, 1,
- )
- return (inv_freq / factor) * ramp + inv_freq * (1 - ramp)
-
- return RotaryEmbedding(
- head_dim,
- rotary_dim,
- max_position,
- base,
- post_process,
- attention_factor=float(attention_factor),
- is_neox=is_neox,
+ ramp = torch.clamp(
+ (torch.arange(rotary_dim // 2, dtype=torch.float32) - low) / (high - low), 0, 1
)
+ return (inv_freq / factor) * ramp + inv_freq * (1 - ramp), float(attention_factor)
raise ValueError(f"Unsupported {rope_scaling = }")
+def _cos_sin_table(
+ head_size: int,
+ rotary_dim: int,
+ positions: int,
+ base: float,
+ rope_scaling: Tuple[Tuple[str, Any], ...] | None,
+ device: torch.device,
+) -> torch.Tensor:
+ inv_freq, attention_factor = _rope_frequencies(
+ head_size, rotary_dim, base, dict(rope_scaling) if rope_scaling is not None else None
+ )
+ freqs = torch.arange(positions, dtype=torch.float)[:, None] * inv_freq[None, :]
+ return torch.cat((freqs.cos() * attention_factor, freqs.sin() * attention_factor), dim=-1)
+
+
+def rope_cos_sin_table(
+ head_size: int,
+ rotary_dim: int,
+ max_position: int,
+ base: float,
+ rope_scaling: Tuple[Tuple[str, Any], ...] | None = None,
+) -> torch.Tensor:
+ """The ``[positions, width]`` cos|sin table, one per distinct rope: only proportional rope
+ depends on the head size, so an indexer whose heads differ in width reads the attention's table."""
+ rope_map = dict(rope_scaling) if rope_scaling is not None else None
+ proportional = rope_map is not None and rope_map["rope_type"] == "proportional"
+ device = torch.get_default_device()
+ if device.type == "cuda" and device.index is None:
+ device = torch.device("cuda", torch.cuda.current_device())
+ return _cos_sin_table(
+ head_size if proportional else rotary_dim,
+ rotary_dim,
+ rope_table_positions(max_position, rope_map),
+ base,
+ rope_scaling,
+ device,
+ )
+
+
_ROPE_DEVICE: torch.device | None = None
@@ -328,17 +321,15 @@ def get_rope(
mrope_section: Tuple[int, ...] | None = None,
mrope_layout: str = "interleaved",
) -> RotaryEmbedding:
- rope_map = dict(rope_scaling) if rope_scaling is not None else None
-
def build() -> RotaryEmbedding:
+ table = rope_cos_sin_table(head_dim, rotary_dim, max_position, base, rope_scaling)
if mrope_section is not None:
- assert rope_map is None or rope_map.get("rope_type", "default") == "default"
return MRotaryEmbedding(
- head_dim, rotary_dim, max_position, base,
+ head_dim, rotary_dim, table,
is_neox=is_neox, mrope_section=tuple(mrope_section),
layout=mrope_layout,
)
- return _get_rope(head_dim, rotary_dim, max_position, base, rope_map, is_neox)
+ return RotaryEmbedding(head_dim, rotary_dim, table, is_neox=is_neox)
t = torch.tensor([])
if t.device == torch.device("meta"):
@@ -358,5 +349,7 @@ __all__ = [
"RotaryEmbedding",
"build_section_table",
"get_rope",
+ "rope_cos_sin_table",
+ "rope_table_positions",
"set_rope_device",
]
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py
index d7b504a7..c5943dce 100644
--- a/python/freetoken/models/config.py
+++ b/python/freetoken/models/config.py
@@ -92,6 +92,13 @@ class RotaryConfig:
mrope_section: list | None = None
mrope_layout: str = "contiguous" # see freetoken.layers.rotary.build_section_table
+ @property
+ def table_positions(self) -> int:
+ """Positions the cos/sin table covers: the longest sequence this rope can serve."""
+ from freetoken.layers.rotary import rope_table_positions
+
+ return rope_table_positions(self.max_position, self.scaling)
+
def mrope_layout_from_rope_params(rope_params: Any) -> str:
"""rope_parameters flags -> layout name: mrope_interleaved_glm, else mrope_interleaved, else contiguous."""
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index ab2fb9b7..23a62f84 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -272,6 +272,17 @@ def parse_args(
help="Data type for model weights and activations. 'auto' will use FP16 for FP32/FP16 models and BF16 for BF16 models.",
)
+ parser.add_argument(
+ "--hf-overrides",
+ type=_json_object,
+ default=None,
+ metavar="JSON",
+ help="JSON object applied to the checkpoint config the model is built from, as vLLM's "
+ "--hf-overrides: a nested config section is updated key by key, any other value is "
+ "replaced whole. A YaRN rope_parameters override extends the servable context to "
+ "original_max_position_embeddings * factor.",
+ )
+
parser.add_argument(
"--tensor-parallel-size",
"--tp-size",
@@ -892,6 +903,7 @@ def parse_args(
image_max_tokens=image_max_tokens,
processor_kwargs=kwargs.pop("mm_processor_kwargs") or {},
)
+ kwargs["hf_overrides"] = kwargs["hf_overrides"] or {}
result = ServerArgs(**kwargs)
logger.info(f"Parsed arguments:\n{result}")
return result, run_shell
diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.py
index b93d3b12..489585e8 100644
--- a/python/freetoken/utils/hf.py
+++ b/python/freetoken/utils/hf.py
@@ -1,9 +1,8 @@
+import copy
import functools
import json
import os
-from typing import Any
-
-from typing import FrozenSet
+from typing import Any, FrozenSet, Mapping
from huggingface_hub import hf_hub_download, snapshot_download
from huggingface_hub.utils import EntryNotFoundError
@@ -235,20 +234,40 @@ def _load_hf_config(model_path: str) -> Any:
return config
-def cached_load_hf_config(model_path: str) -> PretrainedConfig:
+def _with_overrides(config: Any, data: dict, overrides: Mapping[str, Any]) -> dict:
+ merged = dict(data)
+ for key, value in overrides.items():
+ section = getattr(config, key, None)
+ if isinstance(value, Mapping) and isinstance(section, (PretrainedConfig, RawConfigShim)):
+ merged[key] = _with_overrides(section, merged.get(key) or {}, value)
+ else:
+ merged[key] = copy.deepcopy(value)
+ return merged
+
+
+def cached_load_hf_config(
+ model_path: str, overrides: Mapping[str, Any] | None = None
+) -> PretrainedConfig:
+ """A fresh copy of the checkpoint's config with ``overrides`` applied as vLLM's --hf-overrides
+ does: nested config sections update key by key, every other value is replaced whole."""
# A .gguf file (or an FTW dir converted from one) carries its own metadata (no HF
# config.json); return a shim the model registry dispatches on instead of a
# PretrainedConfig.
from freetoken.models.gguf.reader import gguf_config_source
if (gguf_src := gguf_config_source(model_path)) is not None:
+ if overrides:
+ raise ValueError("--hf-overrides applies to a HF config.json; a GGUF carries its own metadata")
from freetoken.models.gguf.config import build_gguf_shim
return build_gguf_shim(gguf_src)
config = _load_hf_config(model_path)
+ data = config.to_dict()
+ if overrides:
+ data = _with_overrides(config, data, overrides)
if isinstance(config, RawConfigShim):
- return RawConfigShim(config.to_dict())
- return type(config)(**config.to_dict())
+ return RawConfigShim(data)
+ return type(config)(**data)
def _weight_allow_patterns(repo_id: str) -> list[str]:
diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.py
index ef7dc7ff..ad07a9dd 100644
--- a/tests/engine/test_attention_backend_matrix.py
+++ b/tests/engine/test_attention_backend_matrix.py
@@ -40,6 +40,8 @@ def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0,
def _model_config(kind):
+ from freetoken.models.config import RotaryConfig
+
mc = SimpleNamespace(
model_type=kind,
single_stream_only=False,
@@ -48,7 +50,7 @@ def _model_config(kind):
has_swa_attention=False,
has_linear_attention=False,
num_layers=4,
- rotary_config=SimpleNamespace(max_position=1024),
+ rotary_config=RotaryConfig(head_dim=128, rotary_dim=128, max_position=1024, base=1e4, scaling=None),
)
if kind == "full":
specs = (_spec("full", AttnType.FULL),)
diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py
index a4e47aa0..073112bb 100644
--- a/tests/engine/test_cache_budget.py
+++ b/tests/engine/test_cache_budget.py
@@ -430,13 +430,17 @@ def test_page_table_width_covers_whole_trailing_pages():
assert w > last_col and w % 32 == 0
-def _generic_rotary_cfg(max_position, override):
+def _generic_rotary_cfg(max_position, override, scaling=None):
from types import SimpleNamespace
+ from freetoken.models.config import RotaryConfig
+
model_config = SimpleNamespace(
single_stream_only=False, is_moe=False, expert_quant="none",
has_swa_attention=False, has_linear_attention=False,
- rotary_config=SimpleNamespace(max_position=max_position),
+ rotary_config=RotaryConfig(
+ head_dim=128, rotary_dim=128, max_position=max_position, base=1e4, scaling=scaling
+ ),
)
class Cfg:
@@ -476,6 +480,19 @@ def test_adjust_config_allows_override_at_rope_table_boundary():
_adjust_config(_generic_rotary_cfg(max_position=1024, override=1024)) # must not raise
+YARN_4X = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 1024}
+
+
+def test_adjust_config_allows_override_to_the_yarn_extended_table():
+ # A YaRN override that leaves max_position_embeddings at the trained length (Qwen3.8's 1M
+ # recipe) serves original * factor positions; the gate must follow the table, not the config.
+ from freetoken.engine.engine import _adjust_config
+
+ _adjust_config(_generic_rotary_cfg(max_position=1024, override=4096, scaling=YARN_4X))
+ with pytest.raises(ValueError, match=r"rope table \(4096 positions\)"):
+ _adjust_config(_generic_rotary_cfg(max_position=1024, override=4097, scaling=YARN_4X))
+
+
def test_adjust_config_rope_gate_exempts_dsv4():
# DSV4 sizes its own rope table from the resolved max_seq_len (_adjust_dsv4_config),
# so the generic gate must not fire even when the override dwarfs max_position.
diff --git a/tests/kernels/test_mrope.py b/tests/kernels/test_mrope.py
index f8402f87..08dff5cb 100644
--- a/tests/kernels/test_mrope.py
+++ b/tests/kernels/test_mrope.py
@@ -32,6 +32,19 @@ def test_section_tables():
build_section_table(SECTION, "diagonal")
+def test_refuses_a_table_wider_than_the_rotary_dims():
+ # A proportional rope's table spans the whole head; mrope takes its width from the table
+ # and indexes the rotary_dim // 2 section table with it, reading past the end.
+ from freetoken.layers.rotary import get_rope
+
+ get_rope.cache_clear()
+ with pytest.raises(AssertionError, match="rotary_dim"):
+ get_rope(
+ head_dim=HEAD, rotary_dim=ROT, max_position=8, base=1e7,
+ rope_scaling=(("rope_type", "proportional"),), mrope_section=SECTION,
+ )
+
+
def _make(mrope: bool, layout: str = "interleaved"):
from freetoken.layers.rotary import get_rope
diff --git a/tests/kernels/test_rotary.py b/tests/kernels/test_rotary.py
index f3e8654f..82ee0153 100644
--- a/tests/kernels/test_rotary.py
+++ b/tests/kernels/test_rotary.py
@@ -101,7 +101,7 @@ def test_yarn_correction_range_matches_hf_where_the_clamp_and_gap_bind():
longest-wavelength dims; flooring the gap at 1 flattens the ramp for a sub-1 gap."""
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
- from freetoken.layers.rotary import get_rope
+ from freetoken.layers.rotary import _rope_frequencies
class _Shim: # duck-typed config for HF's real _compute_yarn_parameters
def __init__(self, head_dim, max_position, rope_parameters):
@@ -122,16 +122,18 @@ def test_yarn_correction_range_matches_hf_where_the_clamp_and_gap_bind():
(128, 1e6, {"factor": 16.0, "beta_fast": 1.15, "beta_slow": 1.0,
"original_max_position_embeddings": 32768, "truncate": False}),
]
+ def table(inv_freq: torch.Tensor, attention_factor: float) -> torch.Tensor:
+ freqs = torch.outer(torch.arange(max_position, dtype=torch.float), inv_freq.float())
+ return torch.cat((freqs.cos() * attention_factor, freqs.sin() * attention_factor), dim=-1)
+
+ # The frequencies, not get_rope's table: YaRN sizes that to original * factor (up to 2M
+ # rows here), and the correction range is a property of the frequencies alone.
for rotary_dim, base, scaling in cases:
- get_rope.cache_clear()
- rope = get_rope(head_dim=rotary_dim, rotary_dim=rotary_dim, max_position=max_position,
- base=base, rope_scaling=(("rope_type", "yarn"), *scaling.items()))
+ ours = table(*_rope_frequencies(rotary_dim, rotary_dim, base, {"rope_type": "yarn", **scaling}))
params = {"rope_type": "yarn", "rope_theta": base, "partial_rotary_factor": 1.0, **scaling}
- inv_freq, attn_factor = ROPE_INIT_FUNCTIONS["yarn"](
- _Shim(rotary_dim, max_position, params), device=torch.device("cpu"))
- freqs = torch.outer(torch.arange(max_position, dtype=torch.float), inv_freq.float())
- expected = torch.cat((freqs.cos() * attn_factor, freqs.sin() * attn_factor), dim=-1)
- torch.testing.assert_close(rope._cos_sin_cache, expected, rtol=0, atol=1e-6)
+ expected = table(*ROPE_INIT_FUNCTIONS["yarn"](
+ _Shim(rotary_dim, max_position, params), device=torch.device("cpu")))
+ torch.testing.assert_close(ours, expected, rtol=0, atol=1e-6)
def test_yarn_mscale_all_dim_zero_falls_back_like_hf():
@@ -160,3 +162,44 @@ def test_yarn_mscale_all_dim_zero_falls_back_like_hf():
expected = 0.1 * math.log(factor) + 1.0 # 1.277, not 1.196 (= 1 + 0.1*0.707*ln 16)
cos = rope._cos_sin_cache[:, :32]
torch.testing.assert_close(cos[0], torch.full_like(cos[0], expected))
+
+
+def _yarn(factor: float, original: int) -> tuple:
+ return (("rope_type", "yarn"), ("factor", factor), ("original_max_position_embeddings", original))
+
+
+def test_yarn_table_covers_the_extended_context():
+ # A checkpoint extended by overriding only its rope parameters (Qwen3.8's 1M recipe) keeps
+ # max_position_embeddings at the trained length; YaRN serves original * factor positions.
+ from freetoken.layers.rotary import get_rope
+
+ get_rope.cache_clear()
+ rope = get_rope(head_dim=64, rotary_dim=64, max_position=64, base=10000.0, rope_scaling=_yarn(4.0, 64))
+ assert rope._cos_sin_cache.shape == (256, 64)
+
+ longer = get_rope(head_dim=64, rotary_dim=64, max_position=512, base=10000.0, rope_scaling=_yarn(4.0, 64))
+ assert longer._cos_sin_cache.shape == (512, 64)
+ torch.testing.assert_close(longer._cos_sin_cache[:256], rope._cos_sin_cache, rtol=0, atol=0)
+
+
+def test_one_table_serves_every_rope_of_the_same_frequencies():
+ # The QSA indexer ropes 128-wide heads with the attention's 64 rotary dims, and mrope reads
+ # the same rows per axis: one table, not a second copy of a 1M-row fp32 tensor.
+ from freetoken.layers.rotary import MRotaryEmbedding, get_rope
+
+ get_rope.cache_clear()
+ attention = get_rope(head_dim=256, rotary_dim=64, max_position=64, base=1e7, rope_scaling=_yarn(4.0, 64))
+ indexer = get_rope(head_dim=128, rotary_dim=64, max_position=64, base=1e7, rope_scaling=_yarn(4.0, 64))
+ mrope = get_rope(
+ head_dim=256, rotary_dim=64, max_position=64, base=1e7, rope_scaling=_yarn(4.0, 64),
+ mrope_section=(11, 11, 10), mrope_layout="interleaved",
+ )
+ assert indexer.head_size == 128 and isinstance(mrope, MRotaryEmbedding)
+ assert attention._cos_sin_cache.shape == (256, 64)
+ table = attention._cos_sin_cache.data_ptr()
+ assert indexer._cos_sin_cache.data_ptr() == table and mrope._cos_sin_cache.data_ptr() == table
+
+ proportional = (("rope_type", "proportional"),)
+ wide = get_rope(head_dim=128, rotary_dim=64, max_position=4, base=1e4, rope_scaling=proportional)
+ narrow = get_rope(head_dim=64, rotary_dim=64, max_position=4, base=1e4, rope_scaling=proportional)
+ assert wide._cos_sin_cache.shape == (4, 128) and narrow._cos_sin_cache.shape == (4, 64)
diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py
index be532eec..f850927b 100644
--- a/tests/models/qwen4_exp/test_config.py
+++ b/tests/models/qwen4_exp/test_config.py
@@ -215,3 +215,54 @@ def test_released_checkpoints_keep_the_dense_projections_bf16(quantization_confi
quant = _quant(_hf_config(quantization_config), tmp_path)
for prefix in DENSE_PREFIXES + BF16_PREFIXES:
assert quant.scheme_for(prefix) is None, prefix
+
+
+# Qwen3.8-Flash-Next model card, "Processing Ultra-Long Texts": the 1M-token override.
+MODEL_CARD_YARN = {
+ "mrope_interleaved": True,
+ "mrope_section": [11, 11, 10],
+ "rope_type": "yarn",
+ "rope_theta": 10000000,
+ "partial_rotary_factor": 0.25,
+ "factor": 4.0,
+ "original_max_position_embeddings": 262144,
+}
+
+
+def _write_checkpoint_config(path, **fields) -> None:
+ import json
+
+ text = {k: v for k, v in vars(_text_config()).items()}
+ config = {
+ "model_type": "qwen4_exp",
+ "architectures": ["Qwen4ExpForConditionalGeneration"],
+ "image_token_id": 248056,
+ "text_config": text,
+ "quantization_config": RADIXARK_NVFP4,
+ **fields,
+ }
+ (path / "config.json").write_text(json.dumps(config))
+
+
+def test_the_engine_serves_the_model_card_1m_recipe(tmp_path):
+ import torch
+
+ from freetoken.distributed import DistributedInfo
+ from freetoken.engine.config import EngineConfig
+ from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig
+
+ _write_checkpoint_config(tmp_path)
+ config = EngineConfig(
+ model_path=str(tmp_path),
+ tp_info=DistributedInfo(rank=0, size=1),
+ dtype=torch.bfloat16,
+ hf_overrides={"text_config": {"rope_parameters": MODEL_CARD_YARN}},
+ mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)),
+ )
+ rotary = config.model_config.rotary_config
+ assert rotary.scaling["rope_type"] == "yarn" and rotary.scaling["factor"] == 4.0 and rotary.mrope_section is None
+ assert rotary.max_position == 262144 and rotary.rotary_dim == 64 and rotary.base == 10000000
+ assert config.max_seq_len == 4 * 262144
+ plain = EngineConfig(model_path=str(tmp_path), tp_info=DistributedInfo(rank=0, size=1), dtype=torch.bfloat16,
+ mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)))
+ assert plain.model_config.rotary_config.scaling is None and plain.max_seq_len == 262144
diff --git a/tests/models/test_qwen3_vl.py b/tests/models/test_qwen3_vl.py
index c45a9f15..6c773d3d 100644
--- a/tests/models/test_qwen3_vl.py
+++ b/tests/models/test_qwen3_vl.py
@@ -39,7 +39,7 @@ def _engine_config(monkeypatch, hf, processor, **overrides):
import freetoken.engine.config as engine_config
import freetoken.mm.processor as mm_processor
- monkeypatch.setattr(engine_config, "cached_load_hf_config", lambda path: hf)
+ monkeypatch.setattr(engine_config, "cached_load_hf_config", lambda path, overrides: hf)
monkeypatch.setattr(engine_config, "checkpoint_quant_config", lambda *args: None)
monkeypatch.setattr(mm_processor, "get_mm_processor", lambda path, mm=None: processor)
return EngineConfig(
diff --git a/tests/utils/test_hf.py b/tests/utils/test_hf.py
new file mode 100644
index 00000000..28630b14
--- /dev/null
+++ b/tests/utils/test_hf.py
@@ -0,0 +1,34 @@
+"""cached_load_hf_config with --hf-overrides, against the same config written into config.json."""
+
+import copy
+import json
+
+import pytest
+
+from freetoken.utils.hf import cached_load_hf_config
+
+
+def _checkpoint(path, model_type: str, dtype: str, rope_parameters: dict) -> str:
+ path.mkdir()
+ text_config = {"hidden_size": 64, "max_position_embeddings": 4096, "rope_parameters": rope_parameters}
+ (path / "config.json").write_text(json.dumps({"model_type": model_type, "dtype": dtype, "text_config": text_config}))
+ return str(path)
+
+
[email protected]("model_type", ["qwen3_5", "a_model_type_transformers_does_not_know"])
+def test_hf_overrides_load_as_if_the_checkpoint_config_said_so(tmp_path, model_type):
+ # vLLM's merge: a nested config section updates key by key, any other value is replaced whole
+ yarn = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 4096}
+ checkpoint = _checkpoint(tmp_path / "checkpoint", model_type, "bfloat16", {"rope_type": "default", "rope_theta": 1e6})
+ edited = _checkpoint(tmp_path / "edited", model_type, "float16", yarn)
+ overrides = {"dtype": "float16", "text_config": {"rope_parameters": yarn}}
+ requested = copy.deepcopy(overrides)
+
+ hf = cached_load_hf_config(checkpoint, overrides)
+ expected = cached_load_hf_config(edited)
+ assert hf.dtype == expected.dtype
+ assert hf.text_config.to_dict() == expected.text_config.to_dict()
+ # transformers fills rope defaults into the rope_parameters it is handed
+ assert overrides == requested
+ # a fresh copy each time: the override never reaches the cached checkpoint config
+ assert cached_load_hf_config(checkpoint).text_config.rope_parameters["rope_type"] == "default"