Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

13.8 KB
Raw Blame
"""v0.71.24 — 2026 model-family recipe expansion (catalog 116 -> 133).

Pure config/data release: 17 new ready-made SFT recipes for the open-weight
models released Feb-Jun 2026 (Qwen3.5 / Qwen3.6 / DeepSeek-V4 / GLM-5.1 /
Kimi-K2.5/K2.6 / MiniMax-M3 / Mistral-Large-3), plus a stale-repo-ID fix for
``glm-5-sft`` (``THUDM/glm-5`` -> ``zai-org/GLM-5``). CI validates each recipe by
``load_config_from_string`` parse only (no network).
"""
from __future__ import annotations

from pathlib import Path

import pytest
import yaml

from soup_cli.config.loader import load_config_from_string
from soup_cli.config.schema import SoupConfig
from soup_cli.recipes.catalog import RECIPES, get_recipe, list_recipes, search_recipes

# ---------------------------------------------------------------------------
# Step A — 17 new SFT recipes (every base verified to resolve on Hugging Face)
# ---------------------------------------------------------------------------

V07124_RECIPE_NAMES = [
    # Qwen3.5 family (Apache-2.0)
    "qwen3.5-0.8b-sft",
    "qwen3.5-2b-sft",
    "qwen3.5-4b-sft",
    "qwen3.5-9b-sft",
    "qwen3.5-27b-sft",
    "qwen3.5-35b-a3b-sft",
    "qwen3.5-122b-a10b-sft",
    "qwen3.5-397b-a17b-sft",
    # Qwen3.6 (Apache-2.0)
    "qwen3.6-27b-sft",
    "qwen3.6-35b-a3b-sft",
    # DeepSeek-V4 (MIT)
    "deepseek-v4-flash-sft",
    "deepseek-v4-pro-sft",
    # GLM (MIT, zai-org)
    "glm-5.1-sft",
    # Kimi (Modified MIT)
    "kimi-k2.5-sft",
    "kimi-k2.6-sft",
    # MiniMax (MiniMax Community License)
    "minimax-m3-sft",
    # Mistral Large 3 was removed in #1145: its repo has no config.json.
]

# name -> expected base repo-ID (the verified Hugging Face repo)
V07124_RECIPE_BASES = {
    "qwen3.5-0.8b-sft": "Qwen/Qwen3.5-0.8B",
    "qwen3.5-2b-sft": "Qwen/Qwen3.5-2B",
    "qwen3.5-4b-sft": "Qwen/Qwen3.5-4B",
    "qwen3.5-9b-sft": "Qwen/Qwen3.5-9B",
    "qwen3.5-27b-sft": "Qwen/Qwen3.5-27B",
    "qwen3.5-35b-a3b-sft": "Qwen/Qwen3.5-35B-A3B",
    "qwen3.5-122b-a10b-sft": "Qwen/Qwen3.5-122B-A10B",
    "qwen3.5-397b-a17b-sft": "Qwen/Qwen3.5-397B-A17B",
    "qwen3.6-27b-sft": "Qwen/Qwen3.6-27B",
    "qwen3.6-35b-a3b-sft": "Qwen/Qwen3.6-35B-A3B",
    "deepseek-v4-flash-sft": "deepseek-ai/DeepSeek-V4-Flash",
    "deepseek-v4-pro-sft": "deepseek-ai/DeepSeek-V4-Pro",
    "glm-5.1-sft": "zai-org/GLM-5.1",
    "kimi-k2.5-sft": "moonshotai/Kimi-K2.5",
    "kimi-k2.6-sft": "moonshotai/Kimi-K2.6",
    "minimax-m3-sft": "MiniMaxAI/MiniMax-M3",
}

# The MoE bases added in v0.71.24 — each recipe must enable moe_lora + aux loss.
# (Scope is intentionally limited to recipes added in THIS release, not every MoE
# recipe in the catalog.)
V07124_MOE_RECIPES = [
    "qwen3.5-35b-a3b-sft",
    "qwen3.5-122b-a10b-sft",
    "qwen3.5-397b-a17b-sft",
    "qwen3.6-35b-a3b-sft",
    "deepseek-v4-flash-sft",
    "deepseek-v4-pro-sft",
    "glm-5.1-sft",
    "kimi-k2.5-sft",
    "kimi-k2.6-sft",
]

# A MoE checkpoint published as a vision-language wrapper (minimax_m3_vl): it loads
# only through SFT's vision path, which never runs the MoE target step, so it
# declares modality: vision and no moe_lora (#1145; #1179 refuses the pair).
V07124_VL_MOE_RECIPES = ["minimax-m3-sft"]


class TestV07124Recipes:
    def test_recipe_count_target(self) -> None:
        # 17 shipped in v0.71.24; #1145 removed mistral-large-3-sft (no config.json).
        assert len(V07124_RECIPE_NAMES) == 16

    def test_docs_models_md_says_why_mistral_large_3_is_not_shipped(self) -> None:
        # #1145: the paragraph is the one place the reason is written down for users.
        docs = Path(__file__).resolve().parents[1] / "docs" / "models.md"
        assert "**Mistral-Large-3 is not shipped as a recipe.**" in docs.read_text(encoding="utf-8")

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_registered(self, name: str) -> None:
        assert name in RECIPES, f"Recipe {name!r} missing from catalog"
        assert get_recipe(name) is not None

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_base_matches_verified_repo(self, name: str) -> None:
        assert RECIPES[name].model == V07124_RECIPE_BASES[name]

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_is_sft(self, name: str) -> None:
        assert RECIPES[name].task == "sft"

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_metadata_well_formed(self, name: str) -> None:
        meta = RECIPES[name]
        assert meta.model, f"{name}: model is empty"
        assert meta.size, f"{name}: size is empty"
        assert meta.tags, f"{name}: tags is empty"
        assert meta.description, f"{name}: description is empty"
        assert meta.yaml_str, f"{name}: yaml_str is empty"

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_yaml_parses_as_soup_config(self, name: str) -> None:
        cfg = load_config_from_string(RECIPES[name].yaml_str)
        assert isinstance(cfg, SoupConfig)
        assert cfg.base == RECIPES[name].model
        assert cfg.task == "sft"

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_yaml_is_safe_loadable(self, name: str) -> None:
        parsed = yaml.safe_load(RECIPES[name].yaml_str)
        assert isinstance(parsed, dict)
        assert "base" in parsed and "task" in parsed
        # The inline ``base:`` / ``task:`` in the YAML must match the RecipeMeta.
        assert parsed["base"] == RECIPES[name].model
        assert parsed["task"] == RECIPES[name].task

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_model_id_no_null_or_whitespace(self, name: str) -> None:
        meta = RECIPES[name]
        assert "\x00" not in meta.model
        assert " " not in meta.model
        # No path-traversal segments or shell metacharacters in a repo id.
        assert ".." not in meta.model, f"path-traversal segment in {meta.model!r}"
        assert not any(c in meta.model for c in ';|&$`<>(){}*?!\\\n\t'), (
            f"shell metacharacter in {meta.model!r}"
        )
        parts = meta.model.split("/")
        assert len(parts) == 2
        assert all(p for p in parts), f"empty component in {meta.model!r}"

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_max_length_within_bounds(self, name: str) -> None:
        cfg = load_config_from_string(RECIPES[name].yaml_str)
        assert 64 <= cfg.data.max_length <= 1_048_576

    @pytest.mark.parametrize("name", V07124_MOE_RECIPES)
    def test_moe_recipes_enable_moe_lora(self, name: str) -> None:
        cfg = load_config_from_string(RECIPES[name].yaml_str)
        assert cfg.training.moe_lora is True, f"{name} should set moe_lora: true"

    @pytest.mark.parametrize("name", V07124_MOE_RECIPES)
    def test_moe_recipes_set_aux_loss_coeff(self, name: str) -> None:
        # Every new MoE recipe sets the aux-loss coefficient explicitly (0.01) so
        # the whole batch is uniformly self-documenting (review HIGH-1).
        cfg = load_config_from_string(RECIPES[name].yaml_str)
        assert abs(cfg.training.moe_aux_loss_coeff - 0.01) < 1e-9, (
            f"{name} should set moe_aux_loss_coeff: 0.01, got "
            f"{cfg.training.moe_aux_loss_coeff}"
        )

    @pytest.mark.parametrize(
        "name",
        sorted(set(V07124_RECIPE_NAMES) - set(V07124_MOE_RECIPES) - set(V07124_VL_MOE_RECIPES)),
    )
    def test_dense_recipes_do_not_set_moe_lora(self, name: str) -> None:
        # The complement of the MoE list: a dense recipe must NOT enable moe_lora
        # (guards against pasting a MoE block into a dense recipe).
        cfg = load_config_from_string(RECIPES[name].yaml_str)
        assert cfg.training.moe_lora is not True, (
            f"{name} is a dense recipe but has moe_lora: true"
        )

    @pytest.mark.parametrize("name", V07124_VL_MOE_RECIPES)
    def test_vl_moe_recipes_load_through_the_vision_path(self, name: str) -> None:
        cfg = load_config_from_string(RECIPES[name].yaml_str)
        assert cfg.modality == "vision"
        assert cfg.training.moe_lora is not True
        assert "moe" in RECIPES[name].tags
        # the vision path loads AutoProcessor, which is remote code for this base
        assert "--trust-remote-code" in RECIPES[name].description

    @pytest.mark.parametrize("name", V07124_MOE_RECIPES)
    def test_moe_recipes_tagged_moe(self, name: str) -> None:
        assert "moe" in RECIPES[name].tags, (
            f"{name} is a MoE recipe but lacks the 'moe' tag"
        )

    @pytest.mark.parametrize("name", [n for n in V07124_RECIPE_NAMES if n.startswith("qwen")])
    def test_qwen_recipes_tagged_qwen(self, name: str) -> None:
        assert "qwen" in RECIPES[name].tags, f"{name} lacks the 'qwen' tag"

    def test_search_by_task_sft_includes_all_new_recipes(self) -> None:
        sft_models = {r.model for r in search_recipes(task="sft")}
        for name in V07124_RECIPE_NAMES:
            assert RECIPES[name].model in sft_models, (
                f"{name} not returned by search_recipes(task='sft')"
            )

    def test_kimi_k2_5_size_known(self) -> None:
        # K2.5 is a ~1T/32B MoE (verified on the model card) — not "N/A".
        assert RECIPES["kimi-k2.5-sft"].size == "1T"

    def test_kimi_k2_6_size_known(self) -> None:
        # K2.6 is a ~1T/32B MoE (verified on the model card).
        assert RECIPES["kimi-k2.6-sft"].size == "1T"

    def test_minimax_recipe_notes_community_license(self) -> None:
        desc = RECIPES["minimax-m3-sft"].description.lower()
        assert "minimax" in desc
        assert "license" in desc
        # commercial-use caveat must be surfaced
        assert "commercial" in desc

    @pytest.mark.parametrize("name", ["kimi-k2.5-sft", "kimi-k2.6-sft"])
    def test_kimi_recipes_note_modified_mit(self, name: str) -> None:
        assert "mit" in RECIPES[name].description.lower()

    def test_deepseek_v4_sizes_intentionally_na(self) -> None:
        # DeepSeek has not publicly confirmed the V4 parameter counts; "N/A" is the
        # honest value. This documents the deliberate choice (vs Kimi K2.5/K2.6,
        # whose ~1T sizes ARE on the card).
        assert RECIPES["deepseek-v4-flash-sft"].size == "N/A"
        assert RECIPES["deepseek-v4-pro-sft"].size == "N/A"

    @pytest.mark.parametrize("name", V07124_RECIPE_NAMES)
    def test_recipe_searchable(self, name: str) -> None:
        # The recipe key is part of the searchable text, so a query for the recipe's
        # own name must return that exact recipe (deterministic — no token heuristic).
        results = search_recipes(query=name)
        assert any(r.model == RECIPES[name].model for r in results), (
            f"{name} not returned by search_recipes(query={name!r})"
        )


# ---------------------------------------------------------------------------
# Step B — stale repo-ID fix: glm-5-sft (THUDM/glm-5 -> zai-org/GLM-5)
# ---------------------------------------------------------------------------


class TestGlm5RepoIdFix:
    def test_glm5_uses_zai_org(self) -> None:
        meta = RECIPES["glm-5-sft"]
        assert meta.model == "zai-org/GLM-5"
        assert "THUDM" not in meta.model

    def test_glm5_yaml_base_updated(self) -> None:
        meta = RECIPES["glm-5-sft"]
        parsed = yaml.safe_load(meta.yaml_str)
        assert parsed["base"] == "zai-org/GLM-5"
        assert "THUDM/glm-5" not in meta.yaml_str

    def test_glm5_old_key_still_exists(self) -> None:
        # The fix RENAMES the base field; it must NOT delete the old recipe key.
        assert "glm-5-sft" in RECIPES, "glm-5-sft was accidentally removed"
        assert "glm-5.1-sft" in RECIPES, "glm-5.1-sft is the new v0.71.24 recipe"

    def test_glm5_and_glm51_are_distinct_entries(self) -> None:
        assert RECIPES["glm-5-sft"].model != RECIPES["glm-5.1-sft"].model

    def test_no_recipe_references_thudm_glm5(self) -> None:
        for name, meta in RECIPES.items():
            assert "THUDM/glm-5" not in meta.model, (
                f"{name}: model still references THUDM/glm-5"
            )
            assert "THUDM/glm-5" not in meta.yaml_str, (
                f"{name}: yaml_str still references THUDM/glm-5"
            )


# ---------------------------------------------------------------------------
# Catalog count — 116 -> 134 (v0.71.30: -> 137)
# ---------------------------------------------------------------------------


class TestCatalogCount:
    def test_total_recipe_count_matches_the_catalog(self) -> None:
        """Renamed 2026-09-02: the old name said 158 while asserting 162.

        It had drifted through 116 -> 134 -> 137 -> 158 -> 162 without the name
        following, so the one thing a reader saw first was the one thing that
        was false. The count itself is pinned in `test_recipes.py` and by
        `test_recipe_count_is_synced.py`; this row is
        the milestone file's own copy and does not need the number twice.
        """
        from tests.recipe_count import EXPECTED_RECIPE_COUNT, recipe_count_hint

        assert len(RECIPES) == EXPECTED_RECIPE_COUNT, recipe_count_hint(len(RECIPES))

    def test_list_recipes_matches_dict(self) -> None:
        assert len(list_recipes()) == len(RECIPES)

    def test_all_recipes_parse(self) -> None:
        from tests.recipe_contracts import load_catalog_config

        # Every row loads or refuses with its explicitly documented reason.
        for name, meta in RECIPES.items():
            cfg = load_catalog_config(meta)
            if cfg is not None:
                assert cfg.base, name

    def test_all_recipes_yaml_base_matches_model(self) -> None:
        # Catalog-wide integrity invariant — the inline ``base:`` must equal
        # RecipeMeta.model for EVERY recipe (catches a half-applied repo-ID fix).
        for name, meta in RECIPES.items():
            parsed = yaml.safe_load(meta.yaml_str)
            assert parsed.get("base") == meta.model, (
                f"{name}: yaml base {parsed.get('base')!r} != model {meta.model!r}"
            )