Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(config): refuse unsupported resident Kimi K2 loads (#2130)

Nurkhan Esenbek69bc8cb5685b
diff --git a/changelog.d/0.76.0/2130.fixed.md b/changelog.d/0.76.0/2130.fixed.md
new file mode 100644
index 00000000..33e2b4df
--- /dev/null
+++ b/changelog.d/0.76.0/2130.fixed.md
@@ -0,0 +1,11 @@
+- Refuse the known resident Hugging Face Kimi K2 bases during Transformers config
+ validation instead of failing after remote-code/model loading. Mark the two
+ affected recipes unavailable, preserve `recipes show`, and refuse `recipes use`
+ before writing a config with usage-error exit 3. This is a behaviour change:
+ these resident configs previously passed validation and failed later at model
+ loading. Base/Instruct name the guarded `training.stream_layers: true` route
+ added by #2107, its tokenizer prerequisites (`--trust-remote-code` and
+ `pip install tiktoken`) and the streaming guide. Evidence is CPU-only parity on
+ a tiny K2-like checkpoint, not full K2/GPU training. Thinking/Instruct-0905 have
+ no supported route yet. K2.5/K2.6 remain outside this boundary. This is the
+ diagnostic option of #2104 (#2130).
diff --git a/docs/models.md b/docs/models.md
index be572683..b3d59e00 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -23,7 +23,7 @@ Soup works with **any** of the **340,000+** text-generation models on [HuggingFa
| **Qwen 2.5** | Qwen2.5-7B-Instruct, Qwen2.5-Coder-32B-Instruct, Qwen2.5-Math-7B-Instruct | 0.5B–72B | Code, math |
| **DeepSeek** | DeepSeek-R1-Distill-Llama-8B, DeepSeek-V3-0324, DeepSeek-V4-Flash/Pro | 1.5B–1.6T | Reasoning (GRPO), code, MoE |
| **GLM** | GLM-4.6, GLM-5, GLM-5.1 | 357B-754B | Chinese + English, MoE |
-| **Kimi** | Kimi-K2, Kimi-K2.5, Kimi-K2.6 | ~1T (MoE) | Long-context agentic, MoE |
+| **Kimi** | Kimi-K2.5, Kimi-K2.6 | ~1T (MoE) | Long-context agentic, MoE |
| **MiniMax** | MiniMax-M2, MiniMax-M3 | 230B–428B | Agentic, MoE (community license) |
| **Phi-4** | Phi-4-14B, Phi-4-mini-reasoning | 3.8B–14B | Compact reasoning |
| **Mistral** | Mistral-7B-Instruct-v0.3, Mistral-Small-24B | 7B–24B | Fast, efficient |
@@ -36,6 +36,30 @@ Soup works with **any** of the **340,000+** text-generation models on [HuggingFa
**Mistral-Large-3 is not shipped as a recipe.** `mistralai` publishes it in Mistral's native layout (`params.json` + `consolidated-*.safetensors`), with no `config.json`, which `AutoConfig` cannot read, and no flag changes that. A Hugging Face-format conversion exists on the Hub as `FriendliAI/Mistral-Large-3-675B-Instruct-2512-BF16`; it is a third party's re-upload, so Soup does not point a recipe at it (#1145).
+**Resident Kimi K2 is refused on the supported Transformers 5.x stack.**
+The known Hugging Face Base, Instruct, Instruct-0905 and Thinking IDs load remote
+code using removed APIs, including `is_torch_fx_available`, and an inference-only
+MoE training path. `trust_remote_code` does not repair that incompatibility.
+This is a behaviour change: these resident configurations previously passed validation
+and failed later at model loading; now config validation stops before model loading.
+For Base and Instruct only, use the guarded `training.stream_layers: true` route
+added by [#2107](https://github.com/MakazhanAlpamys/Soup/pull/2107). It requires
+`--trust-remote-code` for the tokenizer and `pip install tiktoken`; see
+[streaming guide, Kimi K2 paragraph](performance-and-quantization.md#layer-streaming-beta-v0720-nf4-v0722-disk--wider-archs-v0723-preference-losses-v0724)
+for the guard and prerequisites. Its evidence is CPU-only parity on a tiny
+K2-like checkpoint; full K2 weights, training and GPU execution remain unverified.
+Thinking and Instruct-0905 have no supported route yet. This refusal does not
+qualify local copies, other hubs or Unsloth as working routes. K2.5/K2.6 use
+native `kimi_k25` registration and are outside this boundary.
+
+`kimi-k2-sft` and `kimi-k2-thinking-grpo` remain discoverable with an unsupported
+resident-route description. `soup recipes show` preserves their YAML for inspection;
+`soup recipes use` exits 3 (`EXIT_USAGE_ERROR`) with the per-recipe unavailability
+reason and writes no config.
+Catalog snapshots explicitly record this refusal rather than a usable config.
+This implements #2104's diagnostic option; native resident parity and full K2
+weight validation remain separate work.
+
Qwen3.5, Qwen3.6, and Qwen3.8 checkpoints advertise a multimodal conditional-generation
architecture on the Hub, but Soup's catalog recipes are deliberately text-only.
Their explicit `modality: text` selects `AutoModelForCausalLM`, which instantiates the
diff --git a/scripts/generate_recipe_snapshot.py b/scripts/generate_recipe_snapshot.py
index 7e5c1bbe..562dc961 100644
--- a/scripts/generate_recipe_snapshot.py
+++ b/scripts/generate_recipe_snapshot.py
@@ -90,13 +90,18 @@ def build_snapshot() -> dict[str, dict]:
from soup_cli.recipes.catalog import RECIPES
baseline = build_baseline()
- recipes = {
- name: delta(
- flatten(load_config_from_string(recipe.yaml_str).model_dump(mode="json")),
- baseline,
- )
- for name, recipe in sorted(RECIPES.items())
- }
+ recipes = {}
+ for name, recipe in sorted(RECIPES.items()):
+ try:
+ config = load_config_from_string(recipe.yaml_str)
+ except ValueError as exc:
+ if not recipe.unavailable_reason or recipe.unavailable_reason not in str(exc):
+ raise
+ recipes[name] = {"$unavailable": recipe.unavailable_reason}
+ else:
+ if recipe.unavailable_reason:
+ raise ValueError(f"{name} is marked unavailable but its config loads")
+ recipes[name] = delta(flatten(config.model_dump(mode="json")), baseline)
return {"baseline": baseline, "recipes": recipes}
diff --git a/src/soup_cli/commands/recipes.py b/src/soup_cli/commands/recipes.py
index ad539297..4ca331ac 100644
--- a/src/soup_cli/commands/recipes.py
+++ b/src/soup_cli/commands/recipes.py
@@ -9,7 +9,7 @@ from rich.panel import Panel
from rich.syntax import Syntax
from rich.table import Table
-from soup_cli.utils.exit_codes import GateCommand
+from soup_cli.utils.exit_codes import EXIT_USAGE_ERROR, GateCommand
from soup_cli.utils.terminal import for_terminal
console = Console()
@@ -88,6 +88,10 @@ def use(
console.print(f"[red]Recipe not found: {name}[/]")
raise typer.Exit(1)
+ if recipe.unavailable_reason:
+ console.print(f"[red]{for_terminal(recipe.unavailable_reason)}[/]")
+ raise typer.Exit(EXIT_USAGE_ERROR)
+
from soup_cli.migrate.common import validate_output_path
try:
diff --git a/src/soup_cli/config/schema.py b/src/soup_cli/config/schema.py
index 277e8b00..31a30044 100644
--- a/src/soup_cli/config/schema.py
+++ b/src/soup_cli/config/schema.py
@@ -9228,6 +9228,22 @@ class SoupConfig(BaseModel):
f"Use {changes}, or set use_mod: false."
)
+ @model_validator(mode="after")
+ def _reject_resident_kimi_k2(self) -> "SoupConfig":
+ """K2 remote code cannot import or train on the supported 5.x stack."""
+ from soup_cli.utils.kimi import resident_kimi_k2_refusal
+
+ reason = resident_kimi_k2_refusal(
+ self.base, backend=self.backend, hub=self.training.hub,
+ stream_layers=self.training.stream_layers,
+ )
+ if reason is not None:
+ raise ValueError(
+ f"{reason} base={self.base}. Its remote code uses removed APIs "
+ "and an inference-only MoE training path. Choose a compatible base."
+ )
+ return self
+
@model_validator(mode="after")
def _validate_unsloth_sft_modality(self) -> "SoupConfig":
"""#1186: multimodal SFT setup does not use the requested Unsloth backend."""
diff --git a/src/soup_cli/recipes/catalog.py b/src/soup_cli/recipes/catalog.py
index d0dfb7fc..441c7485 100644
--- a/src/soup_cli/recipes/catalog.py
+++ b/src/soup_cli/recipes/catalog.py
@@ -3,6 +3,8 @@
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
+from soup_cli.utils.kimi import KIMI_K2_GUARDED_STREAM_REASON, KIMI_K2_NO_SUPPORTED_ROUTE_REASON
+
@dataclass(frozen=True)
class RecipeMeta:
@@ -14,6 +16,7 @@ class RecipeMeta:
tags: Tuple[str, ...]
description: str
yaml_str: str
+ unavailable_reason: Optional[str] = None
def list_recipes() -> List[RecipeMeta]:
@@ -2809,7 +2812,8 @@ output: ./output
task="sft",
size="N/A",
tags=("kimi", "moonshot", "moe", "long-context"),
- description="Kimi K2 SFT (Moonshot MoE, long-context-aware)",
+ description="Kimi K2 SFT: resident unsupported; guarded streaming only (see recipes use).",
+ unavailable_reason=KIMI_K2_GUARDED_STREAM_REASON,
yaml_str="""\
base: moonshotai/Kimi-K2-Base
task: sft
@@ -2839,7 +2843,8 @@ output: ./output
task="grpo",
size="N/A",
tags=("kimi", "moonshot", "thinking", "grpo", "reasoning"),
- description="Kimi K2 Thinking GRPO reasoning",
+ description="Kimi K2 Thinking GRPO: unsupported resident route; no supported route yet.",
+ unavailable_reason=KIMI_K2_NO_SUPPORTED_ROUTE_REASON,
yaml_str="""\
base: moonshotai/Kimi-K2-Thinking
task: grpo
diff --git a/src/soup_cli/utils/kimi.py b/src/soup_cli/utils/kimi.py
new file mode 100644
index 00000000..e08cec0f
--- /dev/null
+++ b/src/soup_cli/utils/kimi.py
@@ -0,0 +1,31 @@
+"""Known Kimi K2 resident remote-code boundary; no model or network imports."""
+
+import os
+
+KIMI_K2_RESIDENT_REASON = "Resident Kimi K2 is unsupported on Transformers 5.x"
+KIMI_K2_GUARDED_STREAM_REASON = (
+ KIMI_K2_RESIDENT_REASON + ". Use the guarded training.stream_layers: true route "
+ "(#2107), with --trust-remote-code for the tokenizer and pip install tiktoken. "
+ "See docs/performance-and-quantization.md, Kimi K2: "
+ "CPU-only parity on a tiny K2-like checkpoint; full K2/GPU training is unverified."
+)
+KIMI_K2_NO_SUPPORTED_ROUTE_REASON = KIMI_K2_RESIDENT_REASON + ". No supported route yet."
+KIMI_K2_RESIDENT_REASONS = {
+ "moonshotai/kimi-k2-base": KIMI_K2_GUARDED_STREAM_REASON,
+ "moonshotai/kimi-k2-instruct": KIMI_K2_GUARDED_STREAM_REASON,
+ "moonshotai/kimi-k2-instruct-0905": KIMI_K2_NO_SUPPORTED_ROUTE_REASON,
+ "moonshotai/kimi-k2-thinking": KIMI_K2_NO_SUPPORTED_ROUTE_REASON,
+}
+
+
+def resident_kimi_k2_refusal(
+ base: str, *, backend: str, hub: str, stream_layers: bool,
+) -> str | None:
+ """Only qualify known HF IDs on the resident Transformers path."""
+ if backend == "transformers" and hub == "hf" and not stream_layers:
+ reason = KIMI_K2_RESIDENT_REASONS.get(base.casefold())
+ # A real relative local directory can have the same two-component name
+ # as a Hub ID. Do not classify its potentially repaired code by its name.
+ if reason is not None and not os.path.isdir(base):
+ return reason
+ return None
diff --git a/tests/fixtures/recipe_config_snapshots.json b/tests/fixtures/recipe_config_snapshots.json
index 4bf9571d..d9131305 100644
--- a/tests/fixtures/recipe_config_snapshots.json
+++ b/tests/fixtures/recipe_config_snapshots.json
@@ -810,32 +810,10 @@
"training.lr": 1e-05
},
"kimi-k2-sft": {
- "base": "moonshotai/Kimi-K2-Base",
- "data.format": "auto",
- "data.max_length": 8192,
- "data.train": "./data/train.jsonl",
- "training.epochs": 2,
- "training.lora.alpha": 32,
- "training.lora.dropout": 0.0,
- "training.lora.r": 16,
- "training.lr": 0.0001,
- "training.moe_lora": true
+ "$unavailable": "Resident Kimi K2 is unsupported on Transformers 5.x. Use the guarded training.stream_layers: true route (#2107), with --trust-remote-code for the tokenizer and pip install tiktoken. See docs/performance-and-quantization.md, Kimi K2: CPU-only parity on a tiny K2-like checkpoint; full K2/GPU training is unverified."
},
"kimi-k2-thinking-grpo": {
- "base": "moonshotai/Kimi-K2-Thinking",
- "data.format": "auto",
- "data.max_length": 4096,
- "data.train": "./data/reasoning_prompts.jsonl",
- "task": "grpo",
- "training.batch_size": 1,
- "training.epochs": 1,
- "training.gradient_accumulation_steps": 8,
- "training.gradient_checkpointing": true,
- "training.grpo_beta": 0.04,
- "training.lora.alpha": 32,
- "training.lora.r": 16,
- "training.lr": 5e-06,
- "training.verifiable_domain": "math"
+ "$unavailable": "Resident Kimi K2 is unsupported on Transformers 5.x. No supported route yet."
},
"kimi-k2.5-dpo": {
"base": "moonshotai/Kimi-K2.5",
diff --git a/tests/recipe_contracts.py b/tests/recipe_contracts.py
new file mode 100644
index 00000000..e4c55a39
--- /dev/null
+++ b/tests/recipe_contracts.py
@@ -0,0 +1,27 @@
+"""Catalog controls explicitly cover both supported and documented refused rows."""
+
+import re
+
+import pytest
+
+from soup_cli.config.loader import load_config_from_string
+from soup_cli.config.schema import SoupConfig
+from soup_cli.recipes.catalog import RECIPES, RecipeMeta
+
+
+def load_catalog_config(recipe: RecipeMeta) -> SoupConfig | None:
+ """An unavailable row must really refuse; no skip or unchecked exception."""
+ if recipe.unavailable_reason is not None:
+ with pytest.raises(ValueError, match=re.escape(recipe.unavailable_reason)):
+ load_config_from_string(recipe.yaml_str)
+ return None
+ return load_config_from_string(recipe.yaml_str)
+
+
+def load_shipped_config(name: str, text: str) -> SoupConfig | None:
+ """Keep whole shipped-config sweeps strict, including unavailable recipes."""
+ if name.startswith("recipe:"):
+ recipe = RECIPES[name.removeprefix("recipe:")]
+ assert text == recipe.yaml_str
+ return load_catalog_config(recipe)
+ return load_config_from_string(text)
diff --git a/tests/test_issue1230_refuse_ebft.py b/tests/test_issue1230_refuse_ebft.py
index e4757f5f..2ff9e8b5 100644
--- a/tests/test_issue1230_refuse_ebft.py
+++ b/tests/test_issue1230_refuse_ebft.py
@@ -419,16 +419,19 @@ def _shipped_config_texts() -> list[tuple[str, str]]:
def test_every_shipped_recipe_template_and_example_still_loads():
+ from tests.recipe_contracts import load_shipped_config
+
texts = _shipped_config_texts()
assert len(texts) > 200, len(texts)
refused = []
for name, text in texts:
try:
- cfg = load_config_from_string(text)
+ cfg = load_shipped_config(name, text)
except ValueError as exc:
refused.append((name, str(exc)[:200]))
continue
- assert cfg.training.ebft_variant is None, name
+ if cfg is not None:
+ assert cfg.training.ebft_variant is None, name
assert refused == []
diff --git a/tests/test_issue1240_refuse_longlora.py b/tests/test_issue1240_refuse_longlora.py
index d5285441..beebdd2b 100644
--- a/tests/test_issue1240_refuse_longlora.py
+++ b/tests/test_issue1240_refuse_longlora.py
@@ -352,16 +352,19 @@ def _shipped_config_texts() -> list[tuple[str, str]]:
def test_every_shipped_recipe_template_and_example_still_loads():
+ from tests.recipe_contracts import load_shipped_config
+
texts = _shipped_config_texts()
assert len(texts) > 200, len(texts)
refused = []
for name, text in texts:
try:
- cfg = load_config_from_string(text)
+ cfg = load_shipped_config(name, text)
except ValueError as exc:
refused.append((name, str(exc)[:200]))
continue
- assert cfg.training.use_longlora is False, name
+ if cfg is not None:
+ assert cfg.training.use_longlora is False, name
assert refused == []
diff --git a/tests/test_issue1736_unsloth_tip_only_when_it_loads.py b/tests/test_issue1736_unsloth_tip_only_when_it_loads.py
index e46ab196..7951a4d4 100644
--- a/tests/test_issue1736_unsloth_tip_only_when_it_loads.py
+++ b/tests/test_issue1736_unsloth_tip_only_when_it_loads.py
@@ -33,6 +33,7 @@ from soup_cli.config.loader import load_config_from_string
from soup_cli.config.schema import TEMPLATES, SoupConfig
from soup_cli.recipes.catalog import RECIPES
from tests.conftest import strip_ansi
+from tests.recipe_contracts import load_catalog_config
TIP = "Tip: unsloth is installed"
@@ -42,6 +43,12 @@ _SHIPPED = {
}
+def _shipped_config(name):
+ if name.startswith("recipe:"):
+ return load_catalog_config(RECIPES[name.removeprefix("recipe:")])
+ return load_config_from_string(_SHIPPED[name])
+
+
def _raw(task="sft", **training):
return {
"base": "org/m", "task": task, "data": {"train": "./x.jsonl"}, "training": training,
@@ -113,7 +120,9 @@ class TestEveryShippedConfig:
"""A template or recipe either gets the tip and loads with
``backend: unsloth``, or gets neither."""
text = _SHIPPED[name]
- cfg = load_config_from_string(text)
+ cfg = _shipped_config(name)
+ if cfg is None:
+ return
if cfg.backend != "transformers":
pytest.skip("already on another backend: the tip is not offered")
@@ -122,7 +131,9 @@ class TestEveryShippedConfig:
def test_the_second_validation_is_silent_and_stays_off_the_disk(
self, name, monkeypatch, capsys, caplog
):
- cfg = load_config_from_string(_SHIPPED[name])
+ cfg = _shipped_config(name)
+ if cfg is None:
+ return
capsys.readouterr()
caplog.clear()
@@ -158,8 +169,8 @@ def test_some_shipped_configs_are_on_each_side():
"""Guard on the sweep: it must really see both answers."""
answers = [
_loads_on_unsloth(cfg)
- for cfg in (load_config_from_string(text) for text in _SHIPPED.values())
- if cfg.backend == "transformers"
+ for cfg in (_shipped_config(name) for name in _SHIPPED)
+ if cfg is not None and cfg.backend == "transformers"
]
assert answers.count(True) > 100
diff --git a/tests/test_issue621_recipe_snapshot.py b/tests/test_issue621_recipe_snapshot.py
index 6f37793f..cdde9f3b 100644
--- a/tests/test_issue621_recipe_snapshot.py
+++ b/tests/test_issue621_recipe_snapshot.py
@@ -95,7 +95,11 @@ def _live_baseline() -> dict[str, object]:
def _resolve_flat(name: str) -> dict[str, object]:
from soup_cli.config.loader import load_config_from_string
+ from tests.recipe_contracts import load_catalog_config
+ if RECIPES[name].unavailable_reason:
+ assert load_catalog_config(RECIPES[name]) is None
+ return {"$unavailable": RECIPES[name].unavailable_reason}
return flatten(load_config_from_string(RECIPES[name].yaml_str).model_dump(mode="json"))
diff --git a/tests/test_issue795_quantization_unhonoured_tasks.py b/tests/test_issue795_quantization_unhonoured_tasks.py
index c1edf794..1eb9680f 100644
--- a/tests/test_issue795_quantization_unhonoured_tasks.py
+++ b/tests/test_issue795_quantization_unhonoured_tasks.py
@@ -295,9 +295,10 @@ class TestShippedConfigs:
"""#983's lesson: a gate is only safe if nothing Soup ships trips it. The
three ASR recipes set ``quantization: none``."""
from soup_cli.recipes.catalog import RECIPES
+ from tests.recipe_contracts import load_catalog_config
for recipe in RECIPES.values():
- load_config_from_string(recipe.yaml_str)
+ load_catalog_config(recipe)
def test_soup_draft_distill_config_loads_as_none(self):
from soup_cli.commands.draft import _build_distill_config_yaml
diff --git a/tests/test_issue798_moe_flags_per_task.py b/tests/test_issue798_moe_flags_per_task.py
index 8a25e33e..6b2d32e2 100644
--- a/tests/test_issue798_moe_flags_per_task.py
+++ b/tests/test_issue798_moe_flags_per_task.py
@@ -338,9 +338,10 @@ class TestTheFlagsNoTrainerReadsAreRefused:
class TestShippedConfigs:
def test_every_recipe_still_loads(self):
from soup_cli.recipes.catalog import RECIPES
+ from tests.recipe_contracts import load_catalog_config
for name, recipe in RECIPES.items():
- load_config_from_string(recipe.yaml_str), name
+ load_catalog_config(recipe), name
def test_the_moe_recipes_are_the_ones_this_issue_names(self):
"""17: 9 dpo + 8 grpo (including deepseek-v4-pro-dpo and deepseek-v4-pro-grpo)."""
diff --git a/tests/test_issue897_ui_yaml_body_cap.py b/tests/test_issue897_ui_yaml_body_cap.py
index 35409710..dee7bab5 100644
--- a/tests/test_issue897_ui_yaml_body_cap.py
+++ b/tests/test_issue897_ui_yaml_body_cap.py
@@ -236,4 +236,8 @@ def test_every_shipped_recipe_still_validates_through_web_ui(
)
assert response.status_code == 200, f"{name}: {response.text}"
+ if recipe.unavailable_reason:
+ assert response.json()["valid"] is False
+ assert recipe.unavailable_reason in response.text
+ return
assert response.json()["valid"] is True, f"{name}: {response.json()}"
diff --git a/tests/test_kimi_k2_resident.py b/tests/test_kimi_k2_resident.py
new file mode 100644
index 00000000..e9cf12da
--- /dev/null
+++ b/tests/test_kimi_k2_resident.py
@@ -0,0 +1,165 @@
+"""Resident K2 fails at configuration, before remote-code/model construction."""
+
+from dataclasses import replace
+from unittest.mock import Mock
+
+import pytest
+from typer.testing import CliRunner
+
+from soup_cli.config.loader import load_config_from_string
+from soup_cli.recipes.catalog import RECIPES
+from soup_cli.utils.exit_codes import EXIT_USAGE_ERROR
+from tests.conftest import strip_ansi
+
+BASES = ("moonshotai/Kimi-K2-Base", "moonshotai/Kimi-K2-Instruct",
+ "moonshotai/Kimi-K2-Instruct-0905", "moonshotai/Kimi-K2-Thinking")
+REASON = "Resident Kimi K2 is unsupported on Transformers 5.x"
+BLOCKED = ("kimi-k2-sft", "kimi-k2-thinking-grpo")
+
+
+def _assert_route_text(text, base):
+ if base.casefold() in {"moonshotai/kimi-k2-base", "moonshotai/kimi-k2-instruct"}:
+ assert "training.stream_layers: true" in text
+ assert "guarded" in text.lower()
+ assert "not available until it merges" not in text
+ assert "pending" not in text.lower()
+ assert "--trust-remote-code" in text and "tokenizer" in text
+ assert "pip install tiktoken" in text
+ assert "docs/performance-and-quantization.md" in text
+ assert "CPU-only parity on a tiny K2-like checkpoint" in text
+ assert "no supported route yet" not in text.lower()
+ else:
+ assert "no supported route yet" in text.lower()
+ assert "stream_layers" not in text and "#2107" not in text
+
+
[email protected]("base", BASES)
[email protected]("task", ["sft", "grpo"])
+def test_resident_k2_refuses_with_a_specific_remedy(base, task):
+ with pytest.raises(ValueError, match=REASON) as error:
+ load_config_from_string(f"base: {base}\ntask: {task}\ndata: {{train: data.jsonl}}\n")
+ assert base in str(error.value)
+ _assert_route_text(str(error.value), base)
+ assert "remote code" in str(error.value)
+
+
[email protected]("base", ["moonshotai/Kimi-K2.5", "moonshotai/Kimi-K2.6",
+ "deepseek-ai/DeepSeek-V3", "other/Kimi-K2-Base", "./local-k2"])
+def test_unrelated_models_are_not_reclassified(base):
+ cfg = load_config_from_string(f"base: {base}\ndata: {{train: data.jsonl}}\n")
+ assert cfg.base == base
+
+
+def test_a_relative_local_directory_is_not_mistaken_for_the_public_repo(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ (tmp_path / BASES[0]).mkdir(parents=True)
+ cfg = load_config_from_string(f"base: {BASES[0]}\ndata: {{train: data.jsonl}}\n")
+ assert cfg.base == BASES[0]
+
+
[email protected]("base", BASES)
+def test_hub_id_case_does_not_bypass_the_guard(base):
+ with pytest.raises(ValueError, match=REASON) as error:
+ load_config_from_string(f"base: {base.upper()}\ndata: {{train: data.jsonl}}\n")
+ _assert_route_text(str(error.value), base)
+
+
[email protected]("override", ["training: {stream_layers: true, batch_size: 1}",
+ "backend: unsloth", "training: {hub: modelscope}"])
+def test_guard_does_not_claim_other_loading_routes(override):
+ # This is a scope control, not evidence these routes can train K2 today.
+ cfg = load_config_from_string(
+ "base: moonshotai/Kimi-K2-Base\ndata: {train: data.jsonl}\n" + override + "\n",
+ )
+ assert cfg.base == "moonshotai/Kimi-K2-Base"
+
+
[email protected]("base", [BASES[0], BASES[-1]])
[email protected]("mode", ["--dry-run", "--yes"])
+def test_train_cli_refuses_before_any_model_or_config_download(tmp_path, monkeypatch, base, mode):
+ from soup_cli.cli import app
+
+ monkeypatch.chdir(tmp_path)
+ (tmp_path / "soup.yaml").write_text(
+ f"base: {base}\ndata: {{train: data.jsonl, val_split: 0}}\n", encoding="utf-8",
+ )
+ (tmp_path / "data.jsonl").write_text(
+ '{"instruction":"Q", "output":"A"}\n', encoding="utf-8",
+ )
+ build = Mock(side_effect=AssertionError("refused K2 reached trainer construction"))
+ metadata = Mock(side_effect=AssertionError("refused K2 requested model metadata"))
+ monkeypatch.setattr("soup_cli.trainer.dispatch.build_trainer", build)
+ monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", metadata)
+ result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", mode])
+ output = " ".join(strip_ansi(result.output).split())
+ assert result.exit_code == 1, (output, result.exception)
+ assert REASON in output and base in output
+ _assert_route_text(output, base)
+ build.assert_not_called()
+ metadata.assert_not_called()
+
+
[email protected]("name", BLOCKED)
+def test_catalog_marks_refused_recipes_and_does_not_write_them(tmp_path, monkeypatch, name):
+ from soup_cli.cli import app
+
+ recipe = RECIPES[name]
+ assert recipe.unavailable_reason.startswith(REASON)
+ _assert_route_text(recipe.unavailable_reason, recipe.model)
+ if name == "kimi-k2-sft":
+ assert len(recipe.description) <= 100
+ assert "resident unsupported" in recipe.description
+ assert "guarded streaming only" in recipe.description
+ assert "recipes use" in recipe.description
+ else:
+ _assert_route_text(recipe.description, recipe.model)
+ assert "unsupported" in recipe.description.lower()
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setenv("COLUMNS", "1000")
+ runner = CliRunner()
+ shown = runner.invoke(app, ["recipes", "show", name])
+ assert shown.exit_code == 0
+ assert "unsupported" in strip_ansi(shown.output).lower()
+ listed = runner.invoke(app, ["recipes", "list"])
+ assert listed.exit_code == 0
+ assert "unsupported" in strip_ansi(listed.output).lower()
+ used = runner.invoke(app, ["recipes", "use", name, "--yes"])
+ assert used.exit_code == EXIT_USAGE_ERROR == 3
+ output = " ".join(strip_ansi(used.output).split())
+ assert recipe.unavailable_reason in output
+ _assert_route_text(output, recipe.model)
+ assert not (tmp_path / "soup.yaml").exists()
+
+
+def test_catalog_unavailability_is_limited_to_the_two_affected_recipes():
+ assert {name for name, recipe in RECIPES.items() if recipe.unavailable_reason} == set(BLOCKED)
+
+
+def test_snapshot_records_each_recipes_own_route_reason():
+ from scripts.generate_recipe_snapshot import build_snapshot
+
+ snapshot = build_snapshot()
+ for name in BLOCKED:
+ record = snapshot["recipes"][name]
+ assert record == {"$unavailable": RECIPES[name].unavailable_reason}
+ _assert_route_text(record["$unavailable"], RECIPES[name].model)
+
+
+def test_snapshot_does_not_hide_an_unrelated_config_error(monkeypatch):
+ from scripts.generate_recipe_snapshot import build_snapshot
+ from soup_cli.recipes import catalog
+
+ invalid = replace(RECIPES[BLOCKED[0]], yaml_str=RECIPES[BLOCKED[0]].yaml_str + "typo: true\n")
+ monkeypatch.setattr(catalog, "RECIPES", {BLOCKED[0]: invalid})
+ with pytest.raises(ValueError, match="unknown config key"):
+ build_snapshot()
+
+
+def test_snapshot_refuses_a_stale_unavailable_marker(monkeypatch):
+ from scripts.generate_recipe_snapshot import build_snapshot
+ from soup_cli.recipes import catalog
+
+ stale = replace(RECIPES["qwen3.5-4b-sft"], unavailable_reason=REASON)
+ monkeypatch.setattr(catalog, "RECIPES", {"qwen3.5-4b-sft": stale})
+ with pytest.raises(ValueError, match="marked unavailable but its config loads"):
+ build_snapshot()
diff --git a/tests/test_recipes.py b/tests/test_recipes.py
index bf46f048..e9629e47 100644
--- a/tests/test_recipes.py
+++ b/tests/test_recipes.py
@@ -76,13 +76,15 @@ class TestRecipeCatalog:
assert results == []
def test_all_recipes_valid_yaml(self):
- """All recipes contain valid YAML that loads as SoupConfig."""
- from soup_cli.config.loader import load_config_from_string
+ """Each recipe loads or refuses with its documented unavailability reason."""
from soup_cli.recipes.catalog import RECIPES
+ from tests.recipe_contracts import load_catalog_config
for name, recipe in RECIPES.items():
try:
- config = load_config_from_string(recipe.yaml_str)
+ config = load_catalog_config(recipe)
+ if config is None:
+ continue
assert config.base is not None, f"Recipe {name} has no base model"
except Exception as exc:
pytest.fail(f"Recipe '{name}' is invalid: {exc}")
diff --git a/tests/test_recipes_v031.py b/tests/test_recipes_v031.py
index 3fefc696..ad68b95a 100644
--- a/tests/test_recipes_v031.py
+++ b/tests/test_recipes_v031.py
@@ -15,6 +15,7 @@ from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.config.loader import load_config_from_string
from soup_cli.recipes.catalog import RECIPES, get_recipe, search_recipes
+from tests.recipe_contracts import load_catalog_config
runner = CliRunner()
@@ -319,7 +320,9 @@ class TestRecipeCatalog80:
@pytest.mark.parametrize("name", sorted(RECIPES.keys()))
def test_every_recipe_loads_as_soupconfig(self, name: str) -> None:
recipe = RECIPES[name]
- cfg = load_config_from_string(recipe.yaml_str)
+ cfg = load_catalog_config(recipe)
+ if cfg is None:
+ return
assert cfg.base == recipe.model, (
f"Recipe '{name}' base mismatch: meta={recipe.model}, yaml={cfg.base}"
)
@@ -358,7 +361,9 @@ class TestRecipeCatalog80:
def test_every_recipe_default_data_path_is_relative(self, name: str) -> None:
"""Default training data path must be a non-empty relative path."""
recipe = RECIPES[name]
- cfg = load_config_from_string(recipe.yaml_str)
+ cfg = load_catalog_config(recipe)
+ if cfg is None:
+ return
train_path = cfg.data.train
assert train_path, f"{name} has empty/missing data.train"
# Paths starting with / on POSIX or with a Windows drive letter are absolute.
@@ -375,7 +380,9 @@ class TestRecipeCatalog80:
no-op LoRA adapter at training time with no error: a silent failure
mode the catalog must guard against.
"""
- cfg = load_config_from_string(RECIPES[name].yaml_str)
+ cfg = load_catalog_config(RECIPES[name])
+ if cfg is None:
+ return
modules = cfg.training.lora.target_modules
if isinstance(modules, list):
assert modules, f"{name} has empty lora.target_modules list"
@@ -388,7 +395,9 @@ class TestRecipeCatalog80:
@pytest.mark.parametrize("name", sorted(RECIPES.keys()))
def test_every_recipe_max_length_within_bounds(self, name: str) -> None:
"""max_length must be within the schema bounds [64, 1_048_576]."""
- cfg = load_config_from_string(RECIPES[name].yaml_str)
+ cfg = load_catalog_config(RECIPES[name])
+ if cfg is None:
+ return
assert 64 <= cfg.data.max_length <= 1_048_576, (
f"{name} max_length={cfg.data.max_length} out of [64, 1M]"
)
@@ -400,7 +409,9 @@ class TestRecipeCatalog80:
self, name: str
) -> None:
"""GRPO recipes must wire reward_fn and num_generations — required for the trainer."""
- cfg = load_config_from_string(RECIPES[name].yaml_str)
+ cfg = load_catalog_config(RECIPES[name])
+ if cfg is None:
+ return
assert cfg.training.reward_fn is not None, (
f"GRPO recipe '{name}' missing reward_fn"
)
diff --git a/tests/test_shipped_configs_set_gradient_checkpointing.py b/tests/test_shipped_configs_set_gradient_checkpointing.py
index 4b08e295..d8063828 100644
--- a/tests/test_shipped_configs_set_gradient_checkpointing.py
+++ b/tests/test_shipped_configs_set_gradient_checkpointing.py
@@ -13,7 +13,6 @@ from pathlib import Path
import pytest
import yaml
-from soup_cli.config.loader import load_config_from_string
from soup_cli.config.schema import TEMPLATES
from soup_cli.recipes.catalog import RECIPES
from soup_cli.templates import list_templates, load_template
@@ -61,4 +60,6 @@ def test_shipped_config_sets_gradient_checkpointing(source_id, text):
@pytest.mark.parametrize(("source_id", "text"), SHIPPED, ids=[s for s, _ in SHIPPED])
def test_shipped_config_still_loads(source_id, text):
- load_config_from_string(text)
+ from tests.recipe_contracts import load_shipped_config
+
+ load_shipped_config(source_id, text)
diff --git a/tests/test_shipped_configs_set_no_inert_field.py b/tests/test_shipped_configs_set_no_inert_field.py
index bfa97705..b763516f 100644
--- a/tests/test_shipped_configs_set_no_inert_field.py
+++ b/tests/test_shipped_configs_set_no_inert_field.py
@@ -9,17 +9,19 @@ would get no effect from, or a warning they did not write.
from __future__ import annotations
+import re
import warnings
+from dataclasses import replace
import pytest
import yaml
from soup_cli.config.deprecation import SoupConfigDeprecationWarning
-from soup_cli.config.loader import load_config_from_string
from soup_cli.config.schema import SoupConfig, remap_root_level_misplaced_keys
from soup_cli.config.staged_fields import find_staged_config_fields
from soup_cli.recipes.catalog import RECIPES
from soup_cli.templates import list_templates, load_template
+from tests.recipe_contracts import load_shipped_config
from tests.test_issue748_config_fields_reach_a_consumer import KNOWN_UNCONSUMED
@@ -29,21 +31,29 @@ def _shipped() -> list[tuple[str, str]]:
return configs
-def inert_fields_set(text: str) -> list[str]:
+def inert_fields_set(text: str, *, name: str = "") -> list[str]:
"""Every field ``text`` sets that is unconsumed, staged, or a deprecated value."""
loaded = yaml.safe_load(text) or {}
raw = remap_root_level_misplaced_keys(loaded)
found = []
for section in ("training", "data"):
- for name in (raw.get(section) or {}):
- if f"{section}.{name}" in KNOWN_UNCONSUMED:
- found.append(f"{section}.{name} (read by nothing)")
- config = load_config_from_string(text) # the shipped text must load at all
+ for field in (raw.get(section) or {}):
+ if f"{section}.{field}" in KNOWN_UNCONSUMED:
+ found.append(f"{section}.{field} (read by nothing)")
+ config = load_shipped_config(name, text)
# The loader turns deprecation warnings into console lines (#759), so they are
# recorded here from the same construction it performs.
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
- SoupConfig(**loaded)
+ if config is None:
+ # A documented refusal still validates nested fields and emits their
+ # warnings before the root Kimi guard. Keep inspecting all raw fields.
+ reason = RECIPES[name.removeprefix("recipe:")].unavailable_reason
+ assert reason is not None
+ with pytest.raises(ValueError, match=re.escape(reason)):
+ SoupConfig(**loaded)
+ else:
+ SoupConfig(**loaded)
found += [f"{item.path} (staged)" for item in find_staged_config_fields(raw, config=config)]
found += [
f"deprecated value: {w.message}"
@@ -78,7 +88,25 @@ def test_the_enumeration_covers_both_catalogs_exactly():
@pytest.mark.parametrize("name,text", _shipped(), ids=[n for n, _ in _shipped()])
def test_a_shipped_config_sets_no_inert_field(name, text):
- assert inert_fields_set(text) == [], name
+ assert inert_fields_set(text, name=name) == [], name
+
+
[email protected]("name", ["kimi-k2-sft", "kimi-k2-thinking-grpo"])
[email protected]("section,field,value,expected", [
+ ("training", "warmup_auto", True, "training.warmup_auto (read by nothing)"),
+ ("training", "forgetting_stop", True, "training.forgetting_stop (staged)"),
+ ("data", "remove_unused_columns", True, "deprecated value:"),
+])
+def test_refused_recipes_still_check_every_inert_field(
+ monkeypatch, name, section, field, value, expected,
+):
+ recipe = RECIPES[name]
+ raw = yaml.safe_load(recipe.yaml_str)
+ raw.setdefault(section, {})[field] = value
+ text = yaml.safe_dump(raw)
+ monkeypatch.setitem(RECIPES, name, replace(recipe, yaml_str=text))
+ found = inert_fields_set(text, name=f"recipe:{name}")
+ assert any(item.startswith(expected) for item in found), found
class TestTheCheckCanFail:
diff --git a/tests/test_v0510.py b/tests/test_v0510.py
index 50cddde6..bdaf97a4 100644
--- a/tests/test_v0510.py
+++ b/tests/test_v0510.py
@@ -12,7 +12,7 @@ import yaml
from pydantic import ValidationError
from soup_cli.config.loader import load_config_from_string
-from soup_cli.config.schema import TEMPLATES, SoupConfig, TrainingConfig
+from soup_cli.config.schema import TEMPLATES, DataConfig, SoupConfig, TrainingConfig
from soup_cli.recipes.catalog import RECIPES, get_recipe, list_recipes, search_recipes
from soup_cli.utils import hubs as hubs_mod
from soup_cli.utils.hubs import (
@@ -29,6 +29,7 @@ from soup_cli.utils.multipack_sampler import (
MULTIPACK_ARCHITECTURES,
validate_multipack_architecture,
)
+from tests.recipe_contracts import load_catalog_config
# =====================================================================
# Part E — hubs.validate_hub_name
@@ -505,12 +506,16 @@ class TestV0510Recipes:
assert meta.yaml_str
@pytest.mark.parametrize("name", V0510_RECIPE_NAMES)
- def test_recipe_yaml_parses_as_soup_config(self, name: str) -> None:
- yaml_str = RECIPES[name].yaml_str
- cfg = load_config_from_string(yaml_str)
- assert isinstance(cfg, SoupConfig)
- assert cfg.base == RECIPES[name].model
- assert cfg.task == RECIPES[name].task
+ def test_recipe_yaml_loads_or_refuses_with_its_documented_reason(self, name: str) -> None:
+ recipe = RECIPES[name]
+ cfg = load_catalog_config(recipe)
+ parsed = yaml.safe_load(recipe.yaml_str)
+ assert parsed["base"] == recipe.model
+ assert parsed["task"] == recipe.task
+ if cfg is not None:
+ assert isinstance(cfg, SoupConfig)
+ assert cfg.base == recipe.model
+ assert cfg.task == recipe.task
@pytest.mark.parametrize("name", V0510_RECIPE_NAMES)
def test_recipe_yaml_is_safe_loadable(self, name: str) -> None:
@@ -555,16 +560,22 @@ class TestV0510Recipes:
class TestV0510RecipeInvariants:
@pytest.mark.parametrize("name", V0510_RECIPE_NAMES)
def test_max_length_within_bounds(self, name: str) -> None:
- cfg = load_config_from_string(RECIPES[name].yaml_str)
+ recipe = RECIPES[name]
+ cfg = load_catalog_config(recipe)
+ data = cfg.data if cfg else DataConfig(**yaml.safe_load(recipe.yaml_str).get("data", {}))
# Schema bounds: 64 <= max_length <= 1_048_576
- assert 64 <= cfg.data.max_length <= 1_048_576
+ assert 64 <= data.max_length <= 1_048_576
@pytest.mark.parametrize("name", V0510_RECIPE_NAMES)
def test_task_has_required_fields(self, name: str) -> None:
- cfg = load_config_from_string(RECIPES[name].yaml_str)
- if cfg.task == "grpo":
- assert cfg.training.reward_fn is not None
- assert cfg.training.num_generations >= 1
+ recipe = RECIPES[name]
+ cfg = load_catalog_config(recipe)
+ if recipe.task == "grpo":
+ training = cfg.training if cfg else TrainingConfig(
+ **yaml.safe_load(recipe.yaml_str).get("training", {}),
+ )
+ assert training.reward_fn is not None
+ assert training.num_generations >= 1
# =====================================================================
diff --git a/tests/test_v07124.py b/tests/test_v07124.py
index 266228dc..35cac51d 100644
--- a/tests/test_v07124.py
+++ b/tests/test_v07124.py
@@ -302,10 +302,13 @@ class TestCatalogCount:
assert len(list_recipes()) == len(RECIPES)
def test_all_recipes_parse(self) -> None:
- # Defence-in-depth — the whole catalog must still parse after the additions.
+ from tests.recipe_contracts import load_catalog_config
+
+ # Every row loads or refuses with its explicitly documented reason.
for name, meta in RECIPES.items():
- cfg = load_config_from_string(meta.yaml_str)
- assert cfg.base, name
+ cfg = load_catalog_config(meta)
+ if cfg is not None:
+ assert cfg.base, name
def test_all_recipes_yaml_base_matches_model(self) -> None:
# Catalog-wide integrity invariant — the inline ``base:`` must equal