FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(rotary): size the rope table for YaRN and add --hf-overrides (#573)
YevheniiKotyrlo235e2016ddb6
diff --git a/docs/cli.md b/docs/cli.mdindex 0c339439..fb5e613f 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -33,6 +33,7 @@ parsers all resolve automatically from the checkpoint and the GPU.|---|---|---|| `--model-path`, `--model` | required | Local dir, HF repo id, or an FTW dir (auto-detected) || `--served-model-name` | basename of `--model` | Model id reported by `/v1/models` |+| `--hf-overrides` | — | JSON object applied to the checkpoint's config as vLLM's `--hf-overrides`: a nested config section updates key by key, any other value is replaced whole. A YaRN `rope_parameters` override serves `original_max_position_embeddings * factor` positions |### Server & runtimediff --git a/python/freetoken/attention/qsa_sparse.py b/python/freetoken/attention/qsa_sparse.pyindex b07c9523..8ce63c42 100644--- a/python/freetoken/attention/qsa_sparse.py+++ b/python/freetoken/attention/qsa_sparse.py@@ -167,23 +167,22 @@ class QSASparseAttnBackend(BaseAttnBackend):def _index_rope_cache(self) -> torch.Tensor:"""cos/sin table of the indexer rope: same rotary_dim and frequencies as the main- attention, ``head_size`` 128 instead of 256, so it is a separate get_rope instance.+ attention with ``head_size`` 128 instead of 256, so the attention's own table.The table itself (not RotaryEmbedding.forward) because the indexer's norm+rope is onefused kernel and the compressed keys rope at their group's position, not the query's."""if self._index_cos_sin is None:- from freetoken.layers.rotary import get_rope+ from freetoken.layers.rotary import rope_cos_sin_tablerotary = self.rotary_configwith torch.device(self.device):- rope = get_rope(- head_dim=self.index_head_dim,- rotary_dim=rotary.rotary_dim,- max_position=rotary.max_position,- base=rotary.base,- rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,+ self._index_cos_sin = rope_cos_sin_table(+ self.index_head_dim,+ rotary.rotary_dim,+ rotary.max_position,+ rotary.base,+ tuple(rotary.scaling.items()) if rotary.scaling else None,)- self._index_cos_sin = rope._cos_sin_cache.to(self.device)return self._index_cos_sindef _index_rope_rows(self, positions: torch.Tensor) -> torch.Tensor:diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.pyindex 44c5a38e..3e38cb21 100644--- a/python/freetoken/engine/config.py+++ b/python/freetoken/engine/config.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport copyfrom dataclasses import dataclass, field, replacefrom functools import cached_property-from typing import TYPE_CHECKING, List+from typing import TYPE_CHECKING, Any, List, Mappingimport torchfrom freetoken.distributed import DistributedInfo@@ -23,6 +23,8 @@ class EngineConfig:model_path: strtp_info: DistributedInfodtype: torch.dtype+ # --hf-overrides: applied to the checkpoint config the model is built from (cached_load_hf_config)+ hf_overrides: Mapping[str, Any] = field(default_factory=dict)max_running_req: int = 4attention_backend: str = "auto"moe_strategy: str = "auto"@@ -103,7 +105,7 @@ class EngineConfig:@cached_propertydef hf_config(self):- return cached_load_hf_config(self.model_path)+ return cached_load_hf_config(self.model_path, self.hf_overrides)@cached_propertydef model_spec(self) -> ModelSpec:@@ -142,7 +144,7 @@ class EngineConfig:def max_seq_len(self) -> int:if self.max_seq_len_override is not None:return self.max_seq_len_override- return self.model_config.rotary_config.max_position+ return self.model_config.rotary_config.table_positions@propertydef max_forward_len(self) -> int:diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex cc14cf53..e855bcd3 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -1770,18 +1770,17 @@ def _adjust_config(config: EngineConfig):)override("num_page_override", config.num_token_override // config.page_size)- # The rope cos/sin table is baked to rotary_config.max_position, and neither rope kernel+ # The rope cos/sin table covers rotary_config.table_positions, and neither rope kernel# bounds-checks the position it gathers with -- a longer ceiling reads past the table.# DSV4 is exempt: it sizes its own table from the resolved max_seq_len (_adjust_dsv4_config).rotary = getattr(model_config, "rotary_config", None)seq_override = getattr(config, "max_seq_len_override", None)if seq_override is not None and rotary is not None and not is_dsv4:- if seq_override > rotary.max_position:+ if seq_override > rotary.table_positions:raise ValueError(f"--max-seq-len-override {seq_override} exceeds the model's "- f"rope table ({rotary.max_position} positions). Serving past it would read "- "out of bounds; extend the checkpoint's rope_scaling / "- "max_position_embeddings in config.json instead."+ f"rope table ({rotary.table_positions} positions). Serving past it would read "+ "out of bounds; extend the rope with --hf-overrides (YaRN rope_parameters) instead.")# The startup ServerArgs dump is the *requested* config, printed in the frontend processdiff --git a/python/freetoken/layers/rotary.py b/python/freetoken/layers/rotary.pyindex 0d04645d..d0178f3b 100644--- a/python/freetoken/layers/rotary.py+++ b/python/freetoken/layers/rotary.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport functoolsimport math-from typing import Any, Callable, Dict, Tuple+from typing import Any, Mapping, Tupleimport torch@@ -14,50 +14,21 @@ class RotaryEmbedding(StateLessOP):self,head_size: int,rotary_dim: int,- max_position_embeddings: int,- base: float,- post_process: None | Callable[[torch.Tensor], torch.Tensor] = None,- proportional: bool = False,- attention_factor: float = 1.0,+ cos_sin_cache: torch.Tensor,is_neox: bool = True,) -> None:super().__init__()+ assert 0 < rotary_dim <= head_size+ assert rotary_dim % 2 == 0+ assert head_size in [64, 128, 256, 512]self.head_size = head_sizeself.rotary_dim = rotary_dim# NeoX (half-rotation, HF default) vs GPT-J interleaved (adjacent pairs,# ``rope_interleave`` models: GLM MLA lineage). Both underlying kernels# accept the flag; the cos/sin cache layout is identical.self.is_neox = is_neox- if proportional:- assert 0 < rotary_dim <= head_size- assert rotary_dim % 2 == 0- inv_freq = 1.0 / (- base ** (torch.arange(0, head_size, 2, dtype=torch.float) / head_size)- )- if rotary_dim < head_size:- inv_freq[rotary_dim // 2 :] = 0.0- else:- # Standard (NeoX) rope. Supports partial rotary (rotary_dim < head_size):- # rope is applied to the first ``rotary_dim`` dims of each head, the rest pass- # through. Frequencies are spaced over ``rotary_dim`` (matches HF default- # partial rope, e.g. Qwen3.5 partial_rotary_factor, MiniMax-M2's- # ``apply_rotary_pos_emb``). Full rope is rotary_dim == head_size and is- # unaffected. ``head_size`` is passed to flashinfer separately so it rotates- # only the first ``rotary_dim`` dims.- assert 0 < rotary_dim <= head_size- assert rotary_dim % 2 == 0- inv_freq = 1.0 / (- base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim)- )- if post_process is not None:- inv_freq = post_process(inv_freq)- t = torch.arange(max_position_embeddings, dtype=torch.float)- freqs = torch.einsum("i,j -> ij", t, inv_freq)- cos = freqs.cos() * attention_factor- sin = freqs.sin() * attention_factor- # buffer, so don't load/save- self._cos_sin_cache = torch.cat((cos, sin), dim=-1)- assert self.head_size in [64, 128, 256, 512]+ # shared with every rope of the same frequencies (rope_cos_sin_table), so never written+ self._cos_sin_cache = cos_sin_cachefrom freetoken.kernel.backend import is_flashinfer_installed@@ -160,6 +131,7 @@ class MRotaryEmbedding(RotaryEmbedding):) -> None:super().__init__(*args, **kwargs)assert self.is_neox, "mrope is defined on the NeoX half-rotation layout"+ assert self._cos_sin_cache.shape[1] == self.rotary_dim, "mrope reads a rotary_dim-wide table, not a proportional rope's"half = self.rotary_dim // 2assert sum(mrope_section) == half, (mrope_section, half)self._section_table = build_section_table(tuple(mrope_section), layout)@@ -193,56 +165,50 @@ class MRotaryEmbedding(RotaryEmbedding):return query, key-def _get_rope(- head_dim: int,- rotary_dim: int,- max_position: int,- base: float,- rope_scaling: Dict[str, Any] | None = None,- is_neox: bool = True,-) -> RotaryEmbedding:- if rope_scaling is None:- return RotaryEmbedding(head_dim, rotary_dim, max_position, base, is_neox=is_neox)- # need to test some cases:- match rope_scaling["rope_type"]:+def rope_table_positions(max_position: int, rope_scaling: Mapping[str, Any] | None) -> int:+ """Rows the cos/sin table needs: YaRN serves ``original_max_position_embeddings * factor``,+ above ``max_position_embeddings`` when only the rope parameters are overridden."""+ if rope_scaling is None or rope_scaling.get("rope_type") != "yarn":+ return max_position+ extended = int(rope_scaling["original_max_position_embeddings"] * rope_scaling["factor"])+ return max(max_position, extended)+++def _rope_frequencies(+ head_size: int, rotary_dim: int, base: float, rope_scaling: Mapping[str, Any] | None+) -> Tuple[torch.Tensor, float]:+ """``inv_freq`` over the table's half width, and the factor cos and sin are both scaled by."""+ rope_type = "default" if rope_scaling is None else rope_scaling["rope_type"]+ if rope_type == "proportional":+ # Frequencies spaced over the whole head; the pairs past rotary_dim do not rotate.+ inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2, dtype=torch.float) / head_size))+ inv_freq[rotary_dim // 2 :] = 0.0+ return inv_freq, 1.0+ # Standard (NeoX) rope. Supports partial rotary (rotary_dim < head_size): rope is applied+ # to the first ``rotary_dim`` dims of each head, the rest pass through, and frequencies+ # are spaced over ``rotary_dim`` (HF default partial rope, e.g. Qwen3.5+ # partial_rotary_factor, MiniMax-M2's ``apply_rotary_pos_emb``).+ inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))+ match rope_type:case "default":- return RotaryEmbedding(head_dim, rotary_dim, max_position, base, is_neox=is_neox)-- case "proportional":- return RotaryEmbedding(- head_dim,- rotary_dim,- max_position,- base,- proportional=True,- is_neox=is_neox,- )+ return inv_freq, 1.0case "llama3":scaling_factor: float = rope_scaling["factor"]low_freq_factor: float = rope_scaling["low_freq_factor"]high_freq_factor: float = rope_scaling["high_freq_factor"]original_max_position: int = rope_scaling["original_max_position_embeddings"]-- def post_process(inv_freq: torch.Tensor) -> torch.Tensor:- # no smooth if low_freq_factor == high_freq_factor- wave_len = 2 * math.pi / inv_freq- if low_freq_factor == high_freq_factor:- return torch.where(- wave_len < original_max_position / high_freq_factor,- inv_freq,- inv_freq / scaling_factor,- )-- delta = high_freq_factor - low_freq_factor- smooth = (original_max_position / wave_len - low_freq_factor) / delta- smooth = torch.clamp(smooth, 0, 1)- factor = (1 - smooth) / scaling_factor + smooth- return factor * inv_freq-- return RotaryEmbedding(- head_dim, rotary_dim, max_position, base, post_process, is_neox=is_neox- )+ wave_len = 2 * math.pi / inv_freq+ # no smooth if low_freq_factor == high_freq_factor+ if low_freq_factor == high_freq_factor:+ return torch.where(+ wave_len < original_max_position / high_freq_factor,+ inv_freq,+ inv_freq / scaling_factor,+ ), 1.0+ delta = high_freq_factor - low_freq_factor+ smooth = torch.clamp((original_max_position / wave_len - low_freq_factor) / delta, 0, 1)+ return ((1 - smooth) / scaling_factor + smooth) * inv_freq, 1.0case "yarn":factor: float = rope_scaling["factor"]@@ -288,27 +254,54 @@ def _get_rope(high = min(high, rotary_dim - 1)if low == high: # HF nudges instead of flooring the gap at 1 ("truncate": false)high += 0.001-- def post_process(inv_freq: torch.Tensor) -> torch.Tensor:- ramp = torch.clamp(- (torch.arange(rotary_dim // 2, dtype=torch.float32) - low) / (high - low),- 0, 1,- )- return (inv_freq / factor) * ramp + inv_freq * (1 - ramp)-- return RotaryEmbedding(- head_dim,- rotary_dim,- max_position,- base,- post_process,- attention_factor=float(attention_factor),- is_neox=is_neox,+ ramp = torch.clamp(+ (torch.arange(rotary_dim // 2, dtype=torch.float32) - low) / (high - low), 0, 1)+ return (inv_freq / factor) * ramp + inv_freq * (1 - ramp), float(attention_factor)raise ValueError(f"Unsupported {rope_scaling = }")+def _cos_sin_table(+ head_size: int,+ rotary_dim: int,+ positions: int,+ base: float,+ rope_scaling: Tuple[Tuple[str, Any], ...] | None,+ device: torch.device,+) -> torch.Tensor:+ inv_freq, attention_factor = _rope_frequencies(+ head_size, rotary_dim, base, dict(rope_scaling) if rope_scaling is not None else None+ )+ freqs = torch.arange(positions, dtype=torch.float)[:, None] * inv_freq[None, :]+ return torch.cat((freqs.cos() * attention_factor, freqs.sin() * attention_factor), dim=-1)+++def rope_cos_sin_table(+ head_size: int,+ rotary_dim: int,+ max_position: int,+ base: float,+ rope_scaling: Tuple[Tuple[str, Any], ...] | None = None,+) -> torch.Tensor:+ """The ``[positions, width]`` cos|sin table, one per distinct rope: only proportional rope+ depends on the head size, so an indexer whose heads differ in width reads the attention's table."""+ rope_map = dict(rope_scaling) if rope_scaling is not None else None+ proportional = rope_map is not None and rope_map["rope_type"] == "proportional"+ device = torch.get_default_device()+ if device.type == "cuda" and device.index is None:+ device = torch.device("cuda", torch.cuda.current_device())+ return _cos_sin_table(+ head_size if proportional else rotary_dim,+ rotary_dim,+ rope_table_positions(max_position, rope_map),+ base,+ rope_scaling,+ device,+ )++_ROPE_DEVICE: torch.device | None = None@@ -328,17 +321,15 @@ def get_rope(mrope_section: Tuple[int, ...] | None = None,mrope_layout: str = "interleaved",) -> RotaryEmbedding:- rope_map = dict(rope_scaling) if rope_scaling is not None else None-def build() -> RotaryEmbedding:+ table = rope_cos_sin_table(head_dim, rotary_dim, max_position, base, rope_scaling)if mrope_section is not None:- assert rope_map is None or rope_map.get("rope_type", "default") == "default"return MRotaryEmbedding(- head_dim, rotary_dim, max_position, base,+ head_dim, rotary_dim, table,is_neox=is_neox, mrope_section=tuple(mrope_section),layout=mrope_layout,)- return _get_rope(head_dim, rotary_dim, max_position, base, rope_map, is_neox)+ return RotaryEmbedding(head_dim, rotary_dim, table, is_neox=is_neox)t = torch.tensor([])if t.device == torch.device("meta"):@@ -358,5 +349,7 @@ __all__ = ["RotaryEmbedding","build_section_table","get_rope",+ "rope_cos_sin_table",+ "rope_table_positions","set_rope_device",]diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex d7b504a7..c5943dce 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -92,6 +92,13 @@ class RotaryConfig:mrope_section: list | None = Nonemrope_layout: str = "contiguous" # see freetoken.layers.rotary.build_section_table+ @property+ def table_positions(self) -> int:+ """Positions the cos/sin table covers: the longest sequence this rope can serve."""+ from freetoken.layers.rotary import rope_table_positions++ return rope_table_positions(self.max_position, self.scaling)+def mrope_layout_from_rope_params(rope_params: Any) -> str:"""rope_parameters flags -> layout name: mrope_interleaved_glm, else mrope_interleaved, else contiguous."""diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex ab2fb9b7..23a62f84 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -272,6 +272,17 @@ def parse_args(help="Data type for model weights and activations. 'auto' will use FP16 for FP32/FP16 models and BF16 for BF16 models.",)+ parser.add_argument(+ "--hf-overrides",+ type=_json_object,+ default=None,+ metavar="JSON",+ help="JSON object applied to the checkpoint config the model is built from, as vLLM's "+ "--hf-overrides: a nested config section is updated key by key, any other value is "+ "replaced whole. A YaRN rope_parameters override extends the servable context to "+ "original_max_position_embeddings * factor.",+ )+parser.add_argument("--tensor-parallel-size","--tp-size",@@ -892,6 +903,7 @@ def parse_args(image_max_tokens=image_max_tokens,processor_kwargs=kwargs.pop("mm_processor_kwargs") or {},)+ kwargs["hf_overrides"] = kwargs["hf_overrides"] or {}result = ServerArgs(**kwargs)logger.info(f"Parsed arguments:\n{result}")return result, run_shelldiff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.pyindex b93d3b12..489585e8 100644--- a/python/freetoken/utils/hf.py+++ b/python/freetoken/utils/hf.py@@ -1,9 +1,8 @@+import copyimport functoolsimport jsonimport os-from typing import Any--from typing import FrozenSet+from typing import Any, FrozenSet, Mappingfrom huggingface_hub import hf_hub_download, snapshot_downloadfrom huggingface_hub.utils import EntryNotFoundError@@ -235,20 +234,40 @@ def _load_hf_config(model_path: str) -> Any:return config-def cached_load_hf_config(model_path: str) -> PretrainedConfig:+def _with_overrides(config: Any, data: dict, overrides: Mapping[str, Any]) -> dict:+ merged = dict(data)+ for key, value in overrides.items():+ section = getattr(config, key, None)+ if isinstance(value, Mapping) and isinstance(section, (PretrainedConfig, RawConfigShim)):+ merged[key] = _with_overrides(section, merged.get(key) or {}, value)+ else:+ merged[key] = copy.deepcopy(value)+ return merged+++def cached_load_hf_config(+ model_path: str, overrides: Mapping[str, Any] | None = None+) -> PretrainedConfig:+ """A fresh copy of the checkpoint's config with ``overrides`` applied as vLLM's --hf-overrides+ does: nested config sections update key by key, every other value is replaced whole."""# A .gguf file (or an FTW dir converted from one) carries its own metadata (no HF# config.json); return a shim the model registry dispatches on instead of a# PretrainedConfig.from freetoken.models.gguf.reader import gguf_config_sourceif (gguf_src := gguf_config_source(model_path)) is not None:+ if overrides:+ raise ValueError("--hf-overrides applies to a HF config.json; a GGUF carries its own metadata")from freetoken.models.gguf.config import build_gguf_shimreturn build_gguf_shim(gguf_src)config = _load_hf_config(model_path)+ data = config.to_dict()+ if overrides:+ data = _with_overrides(config, data, overrides)if isinstance(config, RawConfigShim):- return RawConfigShim(config.to_dict())- return type(config)(**config.to_dict())+ return RawConfigShim(data)+ return type(config)(**data)def _weight_allow_patterns(repo_id: str) -> list[str]:diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.pyindex ef7dc7ff..ad07a9dd 100644--- a/tests/engine/test_attention_backend_matrix.py+++ b/tests/engine/test_attention_backend_matrix.py@@ -40,6 +40,8 @@ def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0,def _model_config(kind):+ from freetoken.models.config import RotaryConfig+mc = SimpleNamespace(model_type=kind,single_stream_only=False,@@ -48,7 +50,7 @@ def _model_config(kind):has_swa_attention=False,has_linear_attention=False,num_layers=4,- rotary_config=SimpleNamespace(max_position=1024),+ rotary_config=RotaryConfig(head_dim=128, rotary_dim=128, max_position=1024, base=1e4, scaling=None),)if kind == "full":specs = (_spec("full", AttnType.FULL),)diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.pyindex a4e47aa0..073112bb 100644--- a/tests/engine/test_cache_budget.py+++ b/tests/engine/test_cache_budget.py@@ -430,13 +430,17 @@ def test_page_table_width_covers_whole_trailing_pages():assert w > last_col and w % 32 == 0-def _generic_rotary_cfg(max_position, override):+def _generic_rotary_cfg(max_position, override, scaling=None):from types import SimpleNamespace+ from freetoken.models.config import RotaryConfig+model_config = SimpleNamespace(single_stream_only=False, is_moe=False, expert_quant="none",has_swa_attention=False, has_linear_attention=False,- rotary_config=SimpleNamespace(max_position=max_position),+ rotary_config=RotaryConfig(+ head_dim=128, rotary_dim=128, max_position=max_position, base=1e4, scaling=scaling+ ),)class Cfg:@@ -476,6 +480,19 @@ def test_adjust_config_allows_override_at_rope_table_boundary():_adjust_config(_generic_rotary_cfg(max_position=1024, override=1024)) # must not raise+YARN_4X = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 1024}+++def test_adjust_config_allows_override_to_the_yarn_extended_table():+ # A YaRN override that leaves max_position_embeddings at the trained length (Qwen3.8's 1M+ # recipe) serves original * factor positions; the gate must follow the table, not the config.+ from freetoken.engine.engine import _adjust_config++ _adjust_config(_generic_rotary_cfg(max_position=1024, override=4096, scaling=YARN_4X))+ with pytest.raises(ValueError, match=r"rope table \(4096 positions\)"):+ _adjust_config(_generic_rotary_cfg(max_position=1024, override=4097, scaling=YARN_4X))++def test_adjust_config_rope_gate_exempts_dsv4():# DSV4 sizes its own rope table from the resolved max_seq_len (_adjust_dsv4_config),# so the generic gate must not fire even when the override dwarfs max_position.diff --git a/tests/kernels/test_mrope.py b/tests/kernels/test_mrope.pyindex f8402f87..08dff5cb 100644--- a/tests/kernels/test_mrope.py+++ b/tests/kernels/test_mrope.py@@ -32,6 +32,19 @@ def test_section_tables():build_section_table(SECTION, "diagonal")+def test_refuses_a_table_wider_than_the_rotary_dims():+ # A proportional rope's table spans the whole head; mrope takes its width from the table+ # and indexes the rotary_dim // 2 section table with it, reading past the end.+ from freetoken.layers.rotary import get_rope++ get_rope.cache_clear()+ with pytest.raises(AssertionError, match="rotary_dim"):+ get_rope(+ head_dim=HEAD, rotary_dim=ROT, max_position=8, base=1e7,+ rope_scaling=(("rope_type", "proportional"),), mrope_section=SECTION,+ )++def _make(mrope: bool, layout: str = "interleaved"):from freetoken.layers.rotary import get_ropediff --git a/tests/kernels/test_rotary.py b/tests/kernels/test_rotary.pyindex f3e8654f..82ee0153 100644--- a/tests/kernels/test_rotary.py+++ b/tests/kernels/test_rotary.py@@ -101,7 +101,7 @@ def test_yarn_correction_range_matches_hf_where_the_clamp_and_gap_bind():longest-wavelength dims; flooring the gap at 1 flattens the ramp for a sub-1 gap."""from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS- from freetoken.layers.rotary import get_rope+ from freetoken.layers.rotary import _rope_frequenciesclass _Shim: # duck-typed config for HF's real _compute_yarn_parametersdef __init__(self, head_dim, max_position, rope_parameters):@@ -122,16 +122,18 @@ def test_yarn_correction_range_matches_hf_where_the_clamp_and_gap_bind():(128, 1e6, {"factor": 16.0, "beta_fast": 1.15, "beta_slow": 1.0,"original_max_position_embeddings": 32768, "truncate": False}),]+ def table(inv_freq: torch.Tensor, attention_factor: float) -> torch.Tensor:+ freqs = torch.outer(torch.arange(max_position, dtype=torch.float), inv_freq.float())+ return torch.cat((freqs.cos() * attention_factor, freqs.sin() * attention_factor), dim=-1)++ # The frequencies, not get_rope's table: YaRN sizes that to original * factor (up to 2M+ # rows here), and the correction range is a property of the frequencies alone.for rotary_dim, base, scaling in cases:- get_rope.cache_clear()- rope = get_rope(head_dim=rotary_dim, rotary_dim=rotary_dim, max_position=max_position,- base=base, rope_scaling=(("rope_type", "yarn"), *scaling.items()))+ ours = table(*_rope_frequencies(rotary_dim, rotary_dim, base, {"rope_type": "yarn", **scaling}))params = {"rope_type": "yarn", "rope_theta": base, "partial_rotary_factor": 1.0, **scaling}- inv_freq, attn_factor = ROPE_INIT_FUNCTIONS["yarn"](- _Shim(rotary_dim, max_position, params), device=torch.device("cpu"))- freqs = torch.outer(torch.arange(max_position, dtype=torch.float), inv_freq.float())- expected = torch.cat((freqs.cos() * attn_factor, freqs.sin() * attn_factor), dim=-1)- torch.testing.assert_close(rope._cos_sin_cache, expected, rtol=0, atol=1e-6)+ expected = table(*ROPE_INIT_FUNCTIONS["yarn"](+ _Shim(rotary_dim, max_position, params), device=torch.device("cpu")))+ torch.testing.assert_close(ours, expected, rtol=0, atol=1e-6)def test_yarn_mscale_all_dim_zero_falls_back_like_hf():@@ -160,3 +162,44 @@ def test_yarn_mscale_all_dim_zero_falls_back_like_hf():expected = 0.1 * math.log(factor) + 1.0 # 1.277, not 1.196 (= 1 + 0.1*0.707*ln 16)cos = rope._cos_sin_cache[:, :32]torch.testing.assert_close(cos[0], torch.full_like(cos[0], expected))+++def _yarn(factor: float, original: int) -> tuple:+ return (("rope_type", "yarn"), ("factor", factor), ("original_max_position_embeddings", original))+++def test_yarn_table_covers_the_extended_context():+ # A checkpoint extended by overriding only its rope parameters (Qwen3.8's 1M recipe) keeps+ # max_position_embeddings at the trained length; YaRN serves original * factor positions.+ from freetoken.layers.rotary import get_rope++ get_rope.cache_clear()+ rope = get_rope(head_dim=64, rotary_dim=64, max_position=64, base=10000.0, rope_scaling=_yarn(4.0, 64))+ assert rope._cos_sin_cache.shape == (256, 64)++ longer = get_rope(head_dim=64, rotary_dim=64, max_position=512, base=10000.0, rope_scaling=_yarn(4.0, 64))+ assert longer._cos_sin_cache.shape == (512, 64)+ torch.testing.assert_close(longer._cos_sin_cache[:256], rope._cos_sin_cache, rtol=0, atol=0)+++def test_one_table_serves_every_rope_of_the_same_frequencies():+ # The QSA indexer ropes 128-wide heads with the attention's 64 rotary dims, and mrope reads+ # the same rows per axis: one table, not a second copy of a 1M-row fp32 tensor.+ from freetoken.layers.rotary import MRotaryEmbedding, get_rope++ get_rope.cache_clear()+ attention = get_rope(head_dim=256, rotary_dim=64, max_position=64, base=1e7, rope_scaling=_yarn(4.0, 64))+ indexer = get_rope(head_dim=128, rotary_dim=64, max_position=64, base=1e7, rope_scaling=_yarn(4.0, 64))+ mrope = get_rope(+ head_dim=256, rotary_dim=64, max_position=64, base=1e7, rope_scaling=_yarn(4.0, 64),+ mrope_section=(11, 11, 10), mrope_layout="interleaved",+ )+ assert indexer.head_size == 128 and isinstance(mrope, MRotaryEmbedding)+ assert attention._cos_sin_cache.shape == (256, 64)+ table = attention._cos_sin_cache.data_ptr()+ assert indexer._cos_sin_cache.data_ptr() == table and mrope._cos_sin_cache.data_ptr() == table++ proportional = (("rope_type", "proportional"),)+ wide = get_rope(head_dim=128, rotary_dim=64, max_position=4, base=1e4, rope_scaling=proportional)+ narrow = get_rope(head_dim=64, rotary_dim=64, max_position=4, base=1e4, rope_scaling=proportional)+ assert wide._cos_sin_cache.shape == (4, 128) and narrow._cos_sin_cache.shape == (4, 64)diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.pyindex be532eec..f850927b 100644--- a/tests/models/qwen4_exp/test_config.py+++ b/tests/models/qwen4_exp/test_config.py@@ -215,3 +215,54 @@ def test_released_checkpoints_keep_the_dense_projections_bf16(quantization_confiquant = _quant(_hf_config(quantization_config), tmp_path)for prefix in DENSE_PREFIXES + BF16_PREFIXES:assert quant.scheme_for(prefix) is None, prefix+++# Qwen3.8-Flash-Next model card, "Processing Ultra-Long Texts": the 1M-token override.+MODEL_CARD_YARN = {+ "mrope_interleaved": True,+ "mrope_section": [11, 11, 10],+ "rope_type": "yarn",+ "rope_theta": 10000000,+ "partial_rotary_factor": 0.25,+ "factor": 4.0,+ "original_max_position_embeddings": 262144,+}+++def _write_checkpoint_config(path, **fields) -> None:+ import json++ text = {k: v for k, v in vars(_text_config()).items()}+ config = {+ "model_type": "qwen4_exp",+ "architectures": ["Qwen4ExpForConditionalGeneration"],+ "image_token_id": 248056,+ "text_config": text,+ "quantization_config": RADIXARK_NVFP4,+ **fields,+ }+ (path / "config.json").write_text(json.dumps(config))+++def test_the_engine_serves_the_model_card_1m_recipe(tmp_path):+ import torch++ from freetoken.distributed import DistributedInfo+ from freetoken.engine.config import EngineConfig+ from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig++ _write_checkpoint_config(tmp_path)+ config = EngineConfig(+ model_path=str(tmp_path),+ tp_info=DistributedInfo(rank=0, size=1),+ dtype=torch.bfloat16,+ hf_overrides={"text_config": {"rope_parameters": MODEL_CARD_YARN}},+ mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)),+ )+ rotary = config.model_config.rotary_config+ assert rotary.scaling["rope_type"] == "yarn" and rotary.scaling["factor"] == 4.0 and rotary.mrope_section is None+ assert rotary.max_position == 262144 and rotary.rotary_dim == 64 and rotary.base == 10000000+ assert config.max_seq_len == 4 * 262144+ plain = EngineConfig(model_path=str(tmp_path), tp_info=DistributedInfo(rank=0, size=1), dtype=torch.bfloat16,+ mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)))+ assert plain.model_config.rotary_config.scaling is None and plain.max_seq_len == 262144diff --git a/tests/models/test_qwen3_vl.py b/tests/models/test_qwen3_vl.pyindex c45a9f15..6c773d3d 100644--- a/tests/models/test_qwen3_vl.py+++ b/tests/models/test_qwen3_vl.py@@ -39,7 +39,7 @@ def _engine_config(monkeypatch, hf, processor, **overrides):import freetoken.engine.config as engine_configimport freetoken.mm.processor as mm_processor- monkeypatch.setattr(engine_config, "cached_load_hf_config", lambda path: hf)+ monkeypatch.setattr(engine_config, "cached_load_hf_config", lambda path, overrides: hf)monkeypatch.setattr(engine_config, "checkpoint_quant_config", lambda *args: None)monkeypatch.setattr(mm_processor, "get_mm_processor", lambda path, mm=None: processor)return EngineConfig(diff --git a/tests/utils/test_hf.py b/tests/utils/test_hf.pynew file mode 100644index 00000000..28630b14--- /dev/null+++ b/tests/utils/test_hf.py@@ -0,0 +1,34 @@+"""cached_load_hf_config with --hf-overrides, against the same config written into config.json."""++import copy+import json++import pytest++from freetoken.utils.hf import cached_load_hf_config+++def _checkpoint(path, model_type: str, dtype: str, rope_parameters: dict) -> str:+ path.mkdir()+ text_config = {"hidden_size": 64, "max_position_embeddings": 4096, "rope_parameters": rope_parameters}+ (path / "config.json").write_text(json.dumps({"model_type": model_type, "dtype": dtype, "text_config": text_config}))+ return str(path)++[email protected]("model_type", ["qwen3_5", "a_model_type_transformers_does_not_know"])+def test_hf_overrides_load_as_if_the_checkpoint_config_said_so(tmp_path, model_type):+ # vLLM's merge: a nested config section updates key by key, any other value is replaced whole+ yarn = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 4096}+ checkpoint = _checkpoint(tmp_path / "checkpoint", model_type, "bfloat16", {"rope_type": "default", "rope_theta": 1e6})+ edited = _checkpoint(tmp_path / "edited", model_type, "float16", yarn)+ overrides = {"dtype": "float16", "text_config": {"rope_parameters": yarn}}+ requested = copy.deepcopy(overrides)++ hf = cached_load_hf_config(checkpoint, overrides)+ expected = cached_load_hf_config(edited)+ assert hf.dtype == expected.dtype+ assert hf.text_config.to_dict() == expected.text_config.to_dict()+ # transformers fills rope defaults into the rope_parameters it is handed+ assert overrides == requested+ # a fresh copy each time: the override never reaches the cached checkpoint config+ assert cached_load_hf_config(checkpoint).text_config.rope_parameters["rope_type"] == "default"