Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

10.3 KB
Raw Blame
#!/usr/bin/env python3
"""Every auto-selectable catalog model carries a reviewed memory layout.

The installer ranks models by curated priority and checks fit with
model_memory.estimate_model_memory(), which is only as good as the layout the
catalog declares. These checks keep the catalog and the estimator honest:

* every installable entry (except Gemma 4 31B) declares its attention
  layout, recurrent state, pinned GGUF size and the config.json revision it
  came from, and so does every other entry that declares a layout;
* operating contexts fit inside the model's native maximum and meet the 64K
  Hermes floor; an entry below the floor is one whose native maximum is
  below it, and its app compatibility says ODS Talk cannot run it;
* ``vram_required_gb`` (what the dashboard shows) sits just above the
  estimate at the operating context;
* runtime profiles either carry a fleet-measured budget or let the estimator
  compute one from their cache settings.
"""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parents[1]
CATALOG = ROOT / "config" / "model-library.json"
sys.path.insert(0, str(ROOT / "extensions" / "services" / "dashboard-api"))

from model_memory import (  # noqa: E402
    architecture_metadata_complete,
    estimate_model_memory,
    kv_layer_count,
)

HERMES_CONTEXT_FLOOR = 65536
# Gemma 4 31B already runs above the Hermes floor (131K) and its layout was
# not reviewed; it keeps the legacy estimate. The other Gemma 4 entries
# declare their sliding-window layout.
GEMMA_EXEMPT = {"gemma4-31b-q4"}
LAYOUT_FIELDS = (
    "block_count", "attention_head_count_kv",
    "attention_key_length", "attention_value_length", "recurrent_state_bytes",
    "size_bytes", "max_context_length", "architecture_source",
)
SLIDING_WINDOW_FIELDS = (
    "sliding_window", "sliding_window_layer_count", "sliding_window_head_count_kv",
    "sliding_window_key_length", "sliding_window_value_length", "attention_layer_count",
)
TALK_BLOCKING_STATUSES = {
    "blocked", "incompatible", "not_recommended", "not_supported", "unsupported",
    "unsupported_until_revalidated", "not_agent_viable",
}


def _models() -> list[dict]:
    return json.loads(CATALOG.read_text(encoding="utf-8"))["models"]


def _installable(model: dict) -> bool:
    return bool(model.get("gguf_url")) and model.get("install_recommendation", True) is not False


def _selected_models() -> list[dict]:
    return [
        model for model in _models()
        if _installable(model) and str(model.get("source") or "") in {"", "curated"}
        and model["id"] not in GEMMA_EXEMPT
    ]


def _reviewed_models() -> list[dict]:
    """Installable entries plus every other entry that declares a layout."""
    selected = {model["id"] for model in _selected_models()}
    return [
        model for model in _models()
        if model["id"] in selected or "recurrent_state_bytes" in model
    ]


def _ids(models):
    return [model["id"] for model in models]


def test_gemma_exemption_is_explicit():
    gemma = {
        model["id"] for model in _models()
        if model.get("family") == "gemma4" and _installable(model)
        and "recurrent_state_bytes" not in model
    }
    assert gemma == GEMMA_EXEMPT


@pytest.mark.parametrize("model", _reviewed_models(), ids=_ids(_reviewed_models()))
def test_entries_declare_a_reviewed_layout(model):
    missing = [field for field in LAYOUT_FIELDS if field not in model]
    assert not missing, (model["id"], missing)
    source = model["architecture_source"]
    assert re.fullmatch(r"[0-9a-f]{40}", source["revision"]), model["id"]
    assert "/" in source["repo"], model["id"]
    assert architecture_metadata_complete(model), model["id"]
    assert 0 < kv_layer_count(model) <= model["block_count"], model["id"]
    assert model["recurrent_state_bytes"] >= 0
    if "sliding_window" in model:
        # Sliding-window layouts: the attention_* keys describe the
        # full-attention layers; layers beyond both sets reuse another
        # layer's cache (Gemma 4 shared-KV layers) and hold no state.
        missing = [field for field in SLIDING_WINDOW_FIELDS if field not in model]
        assert not missing, (model["id"], missing)
        assert model["recurrent_state_bytes"] == 0, model["id"]
        assert kv_layer_count(model) + model["sliding_window_layer_count"] <= model["block_count"]
        return
    # Hybrid layouts must carry their recurrent state; dense ones have none.
    hybrid = kv_layer_count(model) < model["block_count"]
    assert (model["recurrent_state_bytes"] > 0) == hybrid, model["id"]


@pytest.mark.parametrize("model", _selected_models(), ids=_ids(_selected_models()))
def test_operating_context_fits_the_native_maximum_and_hermes(model):
    assert model["context_length"] <= model["max_context_length"], model["id"]
    assert model["context_length"] >= HERMES_CONTEXT_FLOOR, model["id"]


def test_contexts_below_the_hermes_floor_are_native_limits_marked_for_talk():
    """A catalog context under 64K keeps a model out of Hermes and Pixel.

    It is allowed only when the model's own maximum is under the floor, and
    then the entry must tell ODS Talk up front (app_compatibility), instead of
    Hermes failing with a 502 after the model loads.
    """
    for model in _models():
        if not model.get("gguf_url") or str(model.get("source") or "") not in {"", "curated"}:
            continue
        if int(model.get("context_length") or 0) >= HERMES_CONTEXT_FLOOR:
            continue
        native = int(model.get("max_context_length") or model["context_length"])
        assert native < HERMES_CONTEXT_FLOOR, (model["id"], model["context_length"], native)
        talk = ((model.get("app_compatibility") or {}).get("hermes_talk") or {})
        assert str(talk.get("status") or "").lower() in TALK_BLOCKING_STATUSES, model["id"]


@pytest.mark.parametrize("model", _reviewed_models(), ids=_ids(_reviewed_models()))
def test_declared_vram_tracks_the_estimate(model):
    estimate = estimate_model_memory(model).device_gib
    assert estimate <= model["vram_required_gb"] <= estimate + 3.0, (model["id"], estimate)


@pytest.mark.parametrize("model", _selected_models(), ids=_ids(_selected_models()))
def test_runtime_profiles_state_where_their_budget_comes_from(model):
    for profile in model.get("runtime_profiles") or []:
        if profile.get("estimate_source") == "fleet-validated":
            assert profile.get("estimated_required_gb"), (model["id"], profile["id"])
            continue
        # Estimator-derived profiles carry no hand-written budget: the
        # selector computes it from the profile's cache settings, so the two
        # cannot drift apart.
        assert "estimated_required_gb" not in profile, (model["id"], profile["id"])
        assert profile.get("context_length"), (model["id"], profile["id"])
        env = profile.get("env") or {}
        if env.get("LLAMA_ARG_CACHE_TYPE_V", "f16") not in {"f16", "f32", "bf16"}:
            assert env.get("LLAMA_ARG_FLASH_ATTN") == "on", (model["id"], profile["id"])


def test_selection_priorities_cover_every_memory_class():
    for model in _models():
        selection = model.get("selection")
        if selection is None:
            continue
        assert {"discrete", "unified", "cpu"} <= set(selection), model["id"]
        for memory_class in ("discrete", "unified", "cpu"):
            assert isinstance(selection[memory_class], int) and selection[memory_class] >= 0, model["id"]
        if model.get("runtime") == "vllm":
            # vLLM entries never enter a llama.cpp pick; only --llm-runtime
            # vllm reads their "vllm" priority (tests/test-model-library-vllm.py).
            assert set(selection) == {"discrete", "unified", "cpu", "vllm"}, model["id"]
            assert not any(selection[c] for c in ("discrete", "unified", "cpu")), model["id"]
            continue
        assert set(selection) <= {"discrete", "unified", "cpu", "min_capacity_gib"}, model["id"]
        if model["id"] not in GEMMA_EXEMPT and any(selection[c] for c in ("discrete", "unified", "cpu")):
            assert _installable(model), model["id"]


def test_demoted_models_say_why():
    for model in _models():
        if model.get("install_recommendation") is False and model.get("install_recommendation_reason") is not None:
            assert len(model["install_recommendation_reason"]) > 20, model["id"]
    demoted = {
        "phi4-mini-q4", "phi4-q4", "deepseek-r1-7b-q4", "deepseek-r1-14b-q4",
        "deepseek-r1-32b-q4", "deepseek-r1-70b-q4", "qwen3.5-35b-a3b-q4", "qwen3-30b-a3b-q4",
    }
    by_id = {model["id"]: model for model in _models()}
    for model_id in demoted:
        assert by_id[model_id]["install_recommendation"] is False, model_id
        assert by_id[model_id]["install_recommendation_reason"], model_id
    # Qwen3-30B-A3B's config.json declares 40,960 positions.
    assert by_id["qwen3-30b-a3b-q4"]["context_length"] == 40960
    assert by_id["qwen3-30b-a3b-q4"]["max_context_length"] == 40960



def _qwen_tier_map_files() -> dict[str, set[str]]:
    files = {}
    linux = (ROOT / "installers" / "lib" / "tier-map.sh").read_text(encoding="utf-8")
    linux = linux[linux.index("set_qwen_tier_config()"):linux.index("set_gemma4_tier_config()")]
    files["installers/lib/tier-map.sh"] = set(re.findall(r'GGUF_FILE="([^"]+)"', linux))
    macos = (ROOT / "installers" / "macos" / "lib" / "tier-map.sh").read_text(encoding="utf-8")
    macos = macos[macos.index("set_qwen_tier_config()"):macos.index("set_gemma4_tier_config()")]
    files["installers/macos/lib/tier-map.sh"] = set(re.findall(r'GGUF_FILE="([^"]+)"', macos))
    windows = (ROOT / "installers" / "windows" / "lib" / "tier-map.ps1").read_text(encoding="utf-8")
    windows = windows[windows.index("function Resolve-QwenTierConfig"):windows.index("function Resolve-GemmaTierConfig")]
    files["installers/windows/lib/tier-map.ps1"] = set(re.findall(r'GgufFile\s*=\s*"([^"]+)"', windows))
    return {name: {item for item in found if item} for name, found in files.items()}


def test_tier_map_defaults_are_install_recommendations():
    installable = {model["gguf_file"] for model in _models() if _installable(model)}
    for tier_map, files in _qwen_tier_map_files().items():
        assert files, tier_map
        stale = sorted(files - installable)
        assert not stale, (tier_map, stale)


if __name__ == "__main__":
    raise SystemExit(pytest.main([__file__, "-q"]))