ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
import json
import re
from pathlib import Path
from helpers import record_model_performance
from models import GPUInfo, ModelLibraryResponse
from performance_oracle import (
build_models_payload,
collect_runtime_flags,
current_model_matches,
evaluate_performance,
load_evidence,
model_compatibility_runtime_context,
model_app_compatibility,
model_publisher,
normalize_catalog_entry,
planned_model_context,
rank_pre_download_models,
read_env_file_value,
read_env_value,
read_persisted_env_value,
)
def _gpu(name="NVIDIA GeForce RTX 4060", total_mb=8192, backend="nvidia"):
return GPUInfo(
name=name,
memory_used_mb=1024,
memory_total_mb=total_mb,
memory_percent=12.5,
utilization_percent=0,
temperature_c=40,
gpu_backend=backend,
)
def _model():
return {
"id": "qwen3.5-9b-q4",
"name": "Qwen 3.5 9B",
"gguf_file": "Qwen3.5-9B-Q4_K_M.gguf",
"size_mb": 5760,
"vram_required_gb": 8,
"context_length": 32768,
"specialty": "General",
"description": "Test model",
"quantization": "Q4_K_M",
"llm_model_name": "qwen3.5-9b",
}
def test_phi4_dashboard_defaults_to_fitting_context(data_dir, tmp_path):
install_dir = tmp_path / "ods"
install_dir.mkdir()
raw = next(item for item in _official_model_catalog() if item["id"] == "phi4-mini-q4")
payload = build_models_payload(
_gpu(), None, 0, install_dir, data_dir, catalog=[raw], evidence=[],
downloaded_files_override={},
)
model = next(item for item in payload["models"] if item["id"] == raw["id"])
assert model["contextLength"] == 32768
recommended = [option for option in model["contextOptions"] if option["recommended"]]
assert len(recommended) == 1
assert recommended[0]["contextLength"] == 32768
assert recommended[0]["fitsVram"] is True
maximum = next(option for option in model["contextOptions"] if option["fullContext"])
assert maximum["contextLength"] == 128000
assert maximum["fitsVram"] is False
def test_performance_env_readers_share_matching_quote_contract(monkeypatch, tmp_path):
(tmp_path / ".env").write_text(
"PAIRED='catalog-v2'\n"
"UNMATCHED=catalog-v2'\n",
encoding="utf-8",
)
monkeypatch.delenv("PAIRED", raising=False)
monkeypatch.setenv("PROCESS_ONLY", "'runtime-v2'")
assert read_env_file_value("PAIRED", tmp_path) == "catalog-v2"
assert read_env_file_value("UNMATCHED", tmp_path) == "catalog-v2'"
assert read_env_value("PROCESS_ONLY", tmp_path) == "runtime-v2"
assert read_persisted_env_value("UNMATCHED", tmp_path) == "catalog-v2'"
def test_runtime_flags_read_the_llama_cpp_checkpoint_env_name(tmp_path, monkeypatch):
for key in ("LLAMA_ARG_CHECKPOINT_EVERY_NT", "LLAMA_ARG_CHECKPOINT_EVERY_N_TOKENS",
"LLAMA_CHECKPOINT_EVERY_N_TOKENS"):
monkeypatch.delenv(key, raising=False)
(tmp_path / ".env").write_text("LLAMA_ARG_CHECKPOINT_EVERY_NT=-1\n", encoding="utf-8")
assert collect_runtime_flags(tmp_path)["checkpoint_every_n_tokens"] == "-1"
# Evidence recorded under the former key name still matches.
(tmp_path / ".env").write_text("LLAMA_ARG_CHECKPOINT_EVERY_N_TOKENS=-1\n", encoding="utf-8")
assert collect_runtime_flags(tmp_path)["checkpoint_every_n_tokens"] == "-1"
def _official_model_catalog():
catalog_path = Path(__file__).resolve().parents[4] / "config" / "model-library.json"
return json.loads(catalog_path.read_text(encoding="utf-8"))["models"]
def _compatibility_blocks_release_coverage(entry):
status = str((entry or {}).get("status") or "").strip().lower()
return status in {
"blocked",
"incompatible",
"not_agent_viable",
"not_recommended",
"not_supported",
"unsupported",
"unsupported_until_revalidated",
}
def test_current_model_matches_complete_phi_aliases_and_runtime_prefixes():
catalog = {model["id"]: model for model in _official_model_catalog()}
mini = catalog["phi4-mini-q4"]
full = catalog["phi4-q4"]
cases = [
(mini, full, "phi4-mini-q4"),
(mini, full, "Phi-4 Mini"),
(mini, full, "phi-4-mini"),
(mini, full, "Phi-4-mini-instruct-Q4_K_M.gguf"),
(mini, full, "Phi-4-mini-instruct-Q4_K_M"),
(mini, full, "extra.Phi-4-mini-instruct-Q4_K_M.gguf"),
(mini, full, "user.Phi-4-mini-instruct-Q4_K_M.gguf"),
(mini, full, "/models/Phi-4-mini-instruct-Q4_K_M.gguf"),
(full, mini, "phi4-q4"),
(full, mini, "Phi-4 14B"),
(full, mini, "phi-4"),
(full, mini, "phi-4-Q4_K_M.gguf"),
(full, mini, "phi-4-Q4_K_M"),
(full, mini, "extra.phi-4-Q4_K_M.gguf"),
(full, mini, "user.phi-4-Q4_K_M.gguf"),
(full, mini, r"C:\models\phi-4-Q4_K_M.gguf"),
]
for expected, other, runtime_name in cases:
assert current_model_matches(expected, runtime_name)
assert not current_model_matches(other, runtime_name)
assert current_model_matches(mini, None, mini["gguf_file"])
assert not current_model_matches(full, None, mini["gguf_file"])
assert not current_model_matches(full, "custom.phi-4")
def test_normalize_catalog_entry_tolerates_explicit_null_aliases():
"""A catalog entry may carry an explicit ``"aliases": null`` (JSON null).
``.get("aliases", [])`` only substitutes the default when the key is
absent, so a null value used to raise TypeError and take down every
caller of load_model_catalog. Aliases are still derived from the other
identity fields.
"""
entry = normalize_catalog_entry(
{"id": "m", "gguf": "model-Q4_K_M.gguf", "aliases": None}
)
assert entry is not None
assert "m" in entry["aliases"]
assert "model-Q4_K_M.gguf" in entry["aliases"]
def test_normalize_catalog_entry_keeps_listed_aliases():
entry = normalize_catalog_entry(
{"id": "m", "gguf": "model-Q4_K_M.gguf", "aliases": ["custom-alias"]}
)
assert entry is not None
assert "custom-alias" in entry["aliases"]
def test_real_catalog_phi_models_have_exactly_one_loaded_identity(data_dir, tmp_path):
install_dir = tmp_path / "ods"
install_dir.mkdir()
catalog = _official_model_catalog()
for loaded_model, expected_id in [
("phi-4-mini", "phi4-mini-q4"),
("phi-4", "phi4-q4"),
]:
payload = build_models_payload(
_gpu(),
loaded_model,
0,
install_dir,
data_dir,
catalog=catalog,
evidence=[],
)
loaded_rows = [model["id"] for model in payload["models"] if model["status"] == "loaded"]
assert loaded_rows == [expected_id]
assert payload["currentModel"] == expected_id
def test_unmatched_runtime_model_is_visible_without_borrowing_catalog_metadata(data_dir, tmp_path):
install_dir = tmp_path / "ods"
install_dir.mkdir()
catalog_model = {
**_model(),
"id": "qwen3.6-35b-a3b-ud-q4",
"name": "Qwen 3.6 35B-A3B",
"gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf",
"llm_model_name": "qwen3.6-35b-a3b",
"quantization": "UD-Q4_K_M",
}
runtime_name = "Qwen3.6-35B-A3B-GGUF"
payload = build_models_payload(
_gpu(), runtime_name, 0, install_dir, data_dir, catalog=[catalog_model], evidence=[]
)
loaded = [entry for entry in payload["models"] if entry["status"] == "loaded"]
assert len(loaded) == 1
assert loaded[0]["id"].startswith("runtime-")
assert loaded[0]["name"] == runtime_name
assert loaded[0]["metadata"]["source"] == "runtime"
assert loaded[0]["metadata"]["readable"] is False
for field in ("gguf", "downloadUrl", "size", "sizeGb", "vramRequired",
"estimatedRequired", "contextLength", "quantization", "architecture",
"fitsVram", "fitsCurrentVram", "activationSupport"):
assert loaded[0].get(field) is None, field
assert payload["currentModel"] is None
assert payload["loadedModel"] == runtime_name
assert payload["models"][0]["status"] != "loaded"
ModelLibraryResponse(**payload)
matched = build_models_payload(
_gpu(), "qwen3.6-35b-a3b", 0, install_dir, data_dir,
catalog=[catalog_model], evidence=[],
)
assert [entry["id"] for entry in matched["models"] if entry["status"] == "loaded"] == [catalog_model["id"]]
assert not any(entry["metadata"]["source"] == "runtime" for entry in matched["models"])
def test_benchmark_required_without_measurement_or_evidence(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[_model()],
evidence=[],
)
perf = payload["models"][0]["performance"]
assert perf["source"] == "benchmark_required"
assert perf["tokensPerSec"] is None
assert payload["currentModel"] is None
def test_exact_8gb_model_fits_marketing_8gb_gpu(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(total_mb=8188),
None,
0,
install_dir,
data_dir,
catalog=[_model()],
evidence=[],
)
model = payload["models"][0]
assert model["fitsVram"] is True
assert model["performance"]["source"] == "benchmark_required"
def test_build_models_payload_uses_official_model_library(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
(install_dir / "config").mkdir(parents=True)
(install_dir / "config" / "model-library.json").write_text(json.dumps({
"version": 2,
"models": [
{
"id": "phi4-mini-q4",
"name": "Phi-4 Mini",
"gguf_file": "Phi-4-mini-instruct-Q4_K_M.gguf",
"size_mb": 2490,
"vram_required_gb": 4,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Balanced",
"description": "Compact 128K model.",
"llm_model_name": "phi-4-mini",
},
_model(),
],
}), encoding="utf-8")
payload = build_models_payload(_gpu(), None, 0, install_dir, data_dir, evidence=[])
assert [model["id"] for model in payload["models"]] == ["phi4-mini-q4", "qwen3.5-9b-q4"]
assert payload["models"][0]["gguf"] == "Phi-4-mini-instruct-Q4_K_M.gguf"
assert payload["models"][0]["llmModelName"] == "phi-4-mini"
def test_local_model_payload_prefers_canonical_context_when_upgrade_aliases_diverge(
data_dir,
tmp_path,
):
install_dir = tmp_path / "ods"
models_dir = install_dir / "data" / "models"
models_dir.mkdir(parents=True)
(install_dir / ".env").write_text(
"CTX_SIZE=131072\nMAX_CONTEXT=65536\n",
encoding="utf-8",
)
local_model = models_dir / "LocalUpgrade.gguf"
local_model.write_text("model", encoding="utf-8")
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[],
evidence=[],
downloaded_files_override={local_model.name: local_model},
)
assert payload["models"][0]["contextLength"] == 131072
def test_local_model_payload_skips_invalid_canonical_context_for_legacy_alias(
data_dir,
tmp_path,
):
install_dir = tmp_path / "ods"
models_dir = install_dir / "data" / "models"
models_dir.mkdir(parents=True)
(install_dir / ".env").write_text(
"CTX_SIZE=auto\nMAX_CONTEXT=65536\n",
encoding="utf-8",
)
local_model = models_dir / "LocalUpgrade.gguf"
local_model.write_text("model", encoding="utf-8")
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[],
evidence=[],
downloaded_files_override={local_model.name: local_model},
)
assert payload["models"][0]["contextLength"] == 65536
def test_local_model_payload_prefers_persisted_context_over_stale_process_value(
data_dir,
tmp_path,
monkeypatch,
):
install_dir = tmp_path / "ods"
models_dir = install_dir / "data" / "models"
models_dir.mkdir(parents=True)
(install_dir / ".env").write_text(
"CTX_SIZE=\nMAX_CONTEXT=65536\n",
encoding="utf-8",
)
monkeypatch.setenv("CTX_SIZE", "8192")
local_model = models_dir / "LocalUpgrade.gguf"
local_model.write_text("model", encoding="utf-8")
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[],
evidence=[],
downloaded_files_override={local_model.name: local_model},
)
assert payload["models"][0]["contextLength"] == 65536
def test_model_payload_projects_explicit_app_compatibility(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
catalog = [{
"id": "phi4-mini-q4",
"name": "Phi-4 Mini",
"gguf_file": "Phi-4-mini-instruct-Q4_K_M.gguf",
"size_mb": 2490,
"vram_required_gb": 4,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Balanced",
"description": "Compact 128K model.",
"llm_model_name": "phi-4-mini",
"app_compatibility": {
"openai_chat": {"status": "verified", "reason": "direct chat passed"},
"agent_viability": {
"status": "not_agent_viable",
"reason": "Agent validation failed",
"evidence": "fleet-run/example",
},
"hermes_talk": {"status": "unsupported_until_revalidated", "reason": "Talk proof failed"},
"perplexica": {
"status": "unsupported_until_revalidated",
"reason": "Perplexica probe failed",
"evidence": "fleet-run/perplexica",
},
},
}]
payload = build_models_payload(_gpu(), None, 0, install_dir, data_dir, catalog=catalog, evidence=[])
compatibility = payload["models"][0]["appCompatibility"]
assert compatibility["openaiChat"]["status"] == "verified"
assert compatibility["agentViability"]["status"] == "not_agent_viable"
assert compatibility["agentViability"]["reason"] == "Agent validation failed"
assert compatibility["agentViability"]["evidence"] == "fleet-run/example"
assert compatibility["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert compatibility["hermesTalk"]["reason"] == "Talk proof failed"
assert compatibility["perplexica"]["status"] == "unsupported_until_revalidated"
assert compatibility["perplexica"]["reason"] == "Perplexica probe failed"
assert compatibility["perplexica"]["evidence"] == "fleet-run/perplexica"
assert compatibility["perplexica"]["userMessage"] == (
"This model isn't supported in Perplexica yet. Switch to a recommended model to use Perplexica."
)
TALK_NOT_SUPPORTED_COPY = (
"This model isn't supported in ODS Talk yet. Switch to a recommended model to use ODS Talk."
)
_INTERNAL_COPY_MARKERS = re.compile(
r"fleet|revalidat|release coverage|harness|cycle-\d|\d{4}-\d{2}-\d{2}T|tok/s|websocket|model-ui",
re.IGNORECASE,
)
def test_app_compatibility_user_message_is_generic_by_app_and_status():
internal = (
"Fleet model-UI run 2026-07-16T18-10Z on windows-laptop loaded this model; keep it out "
"of ODS Talk release coverage until revalidated."
)
compatibility = model_app_compatibility({
"app_compatibility": {
"hermes_talk": {"status": "unsupported_until_revalidated", "reason": internal},
"agent_viability": {"status": "not_agent_viable", "reason": internal},
"openai_chat": {"status": "verified", "reason": "direct chat passed"},
"perplexica": {"status": "unsupported_until_revalidated", "reason": internal},
"open_webui": {"status": "verified"},
},
})
assert compatibility["hermesTalk"]["userMessage"] == TALK_NOT_SUPPORTED_COPY
assert compatibility["agentViability"]["userMessage"] == (
"Not verified for agent tasks, so responses may fail. "
"Switch to a recommended model for agent features."
)
assert compatibility["openaiChat"]["userMessage"] == "Verified for chat."
assert compatibility["perplexica"]["userMessage"] == (
"This model isn't supported in Perplexica yet. Switch to a recommended model to use Perplexica."
)
assert compatibility["openWebui"]["userMessage"] == "Verified with Open WebUI."
assert compatibility["pixelAgent"]["userMessage"] == "Not yet tested for Portal agent tasks."
# The internal fleet note stays available to operators and tooling, unchanged.
assert compatibility["hermesTalk"]["reason"] == internal
assert compatibility["hermesTalk"]["status"] == "unsupported_until_revalidated"
def test_app_compatibility_user_note_overrides_generic_copy():
compatibility = model_app_compatibility({
"app_compatibility": {
"hermes_talk": {
"status": "unsupported_until_revalidated",
"reason": "internal fleet note",
"userNote": " Voice replies work, but typed chat is not supported yet. ",
},
"perplexica": {"status": "unsupported_until_revalidated", "user_note": "x" * 281},
},
})
assert compatibility["hermesTalk"]["userMessage"] == (
"Voice replies work, but typed chat is not supported yet."
)
# An oversized note is ignored rather than truncated mid-sentence.
assert compatibility["perplexica"]["userMessage"].startswith(
"This model isn't supported in Perplexica yet."
)
def test_out_of_scope_app_compatibility_reports_untested_copy():
compatibility = model_app_compatibility(
{
"app_compatibility": {
"hermes_talk": {
"status": "unsupported_until_revalidated",
"hostScope": ["tower2"],
"reason": "internal fleet note",
},
},
},
runtime_context={"hosts": ["windows-laptop"]},
)
assert compatibility["hermesTalk"]["status"] == "unknown"
assert compatibility["hermesTalk"]["userMessage"] == "Not yet tested with ODS Talk."
assert compatibility["agentViability"]["userMessage"] == "Not yet tested for agent tasks."
def test_real_granite_talk_block_projects_user_copy_and_keeps_internal_note():
model = next(
model for model in _official_model_catalog() if model["id"] == "granite3.3-2b-instruct-q4"
)
compatibility = model_app_compatibility(model, runtime_context={"hosts": ["tower1"]})
assert compatibility["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert compatibility["agentViability"]["status"] == "not_agent_viable"
assert compatibility["hermesTalk"]["reason"].startswith("Fleet model-UI run")
assert compatibility["hermesTalk"]["userMessage"] == TALK_NOT_SUPPORTED_COPY
assert "Fleet" not in compatibility["agentViability"]["userMessage"]
def test_real_catalog_user_messages_never_carry_internal_fleet_notes():
catalog = _official_model_catalog()
hosts = {"windows-laptop", "tower1"}
for model in catalog:
for entry in (model.get("app_compatibility") or {}).values():
if not isinstance(entry, dict):
continue
for scoped in [entry, *(entry.get("scopedOverrides") or [])]:
scope = scoped.get("hostScope") if isinstance(scoped, dict) else None
hosts.update([scope] if isinstance(scope, str) else [str(host) for host in scope or []])
checked = 0
for model in catalog:
for host in sorted(hosts):
compatibility = model_app_compatibility(model, runtime_context={"hosts": [host]})
for key, entry in compatibility.items():
message = entry["userMessage"]
assert message, (model["id"], host, key)
assert not _INTERNAL_COPY_MARKERS.search(message), (model["id"], host, key, message)
if entry.get("reason"):
assert entry["reason"] not in message, (model["id"], host, key)
checked += 1
assert checked > 100
def test_scoped_app_compatibility_applies_only_to_matching_runtime():
model = {
"id": "mistral-nemo-12b-instruct-q4",
"app_compatibility": {
"hermes_talk": {
"status": "unsupported_until_revalidated",
"reason": "Mistral Talk probe failed on Apple llama-server",
"gpuBackendScope": ["apple"],
"llmBackendScope": ["llama-server"],
},
},
}
apple_llama = model_app_compatibility(
model,
runtime_context={"gpuBackend": "apple", "llmBackend": "llama-server", "runtime": "llama-server"},
)
lemonade_amd = model_app_compatibility(
model,
runtime_context={"gpuBackend": "amd", "llmBackend": "lemonade", "runtime": "lemonade"},
)
assert apple_llama["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert apple_llama["agentViability"]["status"] == "not_agent_viable"
assert lemonade_amd["hermesTalk"]["status"] == "unknown"
assert lemonade_amd["agentViability"]["status"] == "unknown"
def test_unmigrated_lemonade_env_reads_as_llama_server(tmp_path, monkeypatch):
# Round F serves the model through llama-server before the installer
# rewrites a Lemonade-era .env: llama-server evidence applies to it, and
# evidence recorded on Lemonade never does (contract section 6.6).
for key in ("ODS_MODE", "LLM_BACKEND", "GPU_BACKEND"):
monkeypatch.delenv(key, raising=False)
install_dir = tmp_path / "ods"
install_dir.mkdir()
(install_dir / ".env").write_text(
"ODS_MODE=lemonade\nGPU_BACKEND=amd\nLLM_BACKEND=lemonade\n", encoding="utf-8",
)
model = {
"id": "scoped-model",
"app_compatibility": {
"hermes_talk": {
"status": "unsupported_until_revalidated",
"reason": "Talk probe failed on llama-server",
"llmBackendScope": ["llama-server"],
},
"perplexica": {
"status": "unsupported_until_revalidated",
"reason": "Perplexica probe failed through Lemonade",
"llmBackendScope": ["lemonade"],
},
},
}
context = model_compatibility_runtime_context(install_dir=install_dir)
verdicts = model_app_compatibility(model, runtime_context=context)
assert (context["llmBackend"], context["runtime"], context["odsMode"]) == ("llama-server", "llama-server", "local")
assert verdicts["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert verdicts["perplexica"]["status"] == "unknown"
def test_host_scoped_app_compatibility_applies_only_to_matching_host():
model = {
"id": "granite4.1-3b-q4",
"app_compatibility": {
"hermes_talk": {
"status": "unsupported_until_revalidated",
"reason": "Granite Talk probe timed out on windows-laptop",
"hostScope": ["windows-laptop"],
},
},
}
windows_laptop = model_app_compatibility(
model,
runtime_context={"host": "windows-laptop", "hosts": ["windows-laptop", "light-worker"]},
)
strixy = model_app_compatibility(
model,
runtime_context={"host": "strixy", "hosts": ["strixy"]},
)
assert windows_laptop["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert windows_laptop["agentViability"]["status"] == "not_agent_viable"
assert strixy["hermesTalk"]["status"] == "unknown"
assert strixy["agentViability"]["status"] == "unknown"
def test_host_scoped_positive_override_preserves_global_negative_elsewhere():
model = {
"id": "smollm3-3b-q4",
"app_compatibility": {
"perplexica": {
"status": "unsupported_until_revalidated",
"reason": "Earlier cross-platform challenge failure",
"scopedOverrides": [
{
"status": "verified",
"label": "Perplexica verified on Strixy",
"reason": "Fresh exact Strixy proof",
"hostScope": ["strixy"],
"llmBackendScope": ["lemonade"],
"expiresAt": "2999-01-01T00:00:00Z",
},
{"status": "verified", "reason": "Unsafe unscoped result"},
],
},
},
}
strixy = model_app_compatibility(
model, runtime_context={"hosts": ["strixy"], "llmBackend": "lemonade"},
)
wrong_backend = model_app_compatibility(
model, runtime_context={"hosts": ["strixy"], "llmBackend": "llama-server"},
)
tower2 = model_app_compatibility(
model, runtime_context={"hosts": ["tower2"], "llmBackend": "lemonade"},
)
assert strixy["perplexica"] == {
"status": "verified",
"label": "Perplexica verified on Strixy",
"reason": "Fresh exact Strixy proof",
"userMessage": "Verified with Perplexica.",
}
assert wrong_backend["perplexica"]["status"] == "unsupported_until_revalidated"
assert tower2["perplexica"]["status"] == "unsupported_until_revalidated"
def test_host_scoped_positive_override_must_be_current_and_dated():
model = {
"app_compatibility": {
"perplexica": {
"status": "unsupported_until_revalidated",
"scopedOverrides": [
{"status": "verified", "hostScope": ["strixy"],
"expiresAt": "2020-01-01T00:00:00Z"},
{"status": "verified", "hostScope": ["strixy"],
"expiresAt": "not-a-date"},
{"status": "verified", "hostScope": ["strixy"]},
],
},
},
}
context = {"hosts": ["strixy"], "llmBackend": "lemonade"}
assert model_app_compatibility(model, runtime_context=context)["perplexica"]["status"] == (
"unsupported_until_revalidated"
)
model["app_compatibility"]["perplexica"]["scopedOverrides"].append({
"status": "verified",
"hostScope": ["strixy"],
"expiresAt": "2999-01-01T00:00:00Z",
})
assert model_app_compatibility(model, runtime_context=context)["perplexica"]["status"] == "verified"
def test_real_smollm3_strixy_verdict_recorded_on_lemonade_no_longer_applies():
# The Strixy Perplexica proof ran through Lemonade; round F invalidates
# evidence recorded on it (contract section 6.6), so the global verdict
# holds on every host until Strixy is revalidated on llama-server.
model = next(
model for model in _official_model_catalog() if model["id"] == "smollm3-3b-q4"
)
assert "scopedOverrides" not in model["app_compatibility"]["perplexica"]
for hosts, backend in ((["strixy"], "llama-server"), (["strixy"], "lemonade"),
(["windows-laptop"], "llama-server"), (["tower2"], "llama-server")):
verdict = model_app_compatibility(model, runtime_context={"hosts": hosts, "llmBackend": backend})
assert verdict["perplexica"]["status"] == "unsupported_until_revalidated", (hosts, backend)
def test_pixel_compatibility_is_explicit_and_host_scoped():
model = {
"id": "pixel-probe-model",
"app_compatibility": {
"agent_viability": {"status": "verified"},
"pixel_agent": {
"status": "not_agent_viable",
"label": "Pixel capability blocked on Windows",
"reason": "A real Pixel tool-loop probe failed.",
"hostScope": ["windows-laptop"],
},
},
}
windows_laptop = model_app_compatibility(
model,
runtime_context={"host": "windows-laptop", "hosts": ["windows-laptop"]},
)
tower2 = model_app_compatibility(
model,
runtime_context={"host": "tower2", "hosts": ["tower2"]},
)
assert windows_laptop["agentViability"]["status"] == "verified"
assert windows_laptop["pixelAgent"]["status"] == "not_agent_viable"
assert "real Pixel tool-loop probe" in windows_laptop["pixelAgent"]["reason"]
assert "agent-viability" not in windows_laptop
assert "pixel-agent" not in windows_laptop
assert tower2["agentViability"]["status"] == "verified"
assert tower2["pixelAgent"]["status"] == "unknown"
def test_missing_pixel_compatibility_is_never_inferred_from_generic_agent_status():
compatibility = model_app_compatibility({
"id": "generic-only",
"app_compatibility": {"agent_viability": {"status": "verified"}},
})
assert compatibility["agentViability"]["status"] == "verified"
assert compatibility["pixelAgent"]["status"] == "unknown"
def test_model_payload_applies_scoped_app_compatibility_from_install_env(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
model = {
"id": "mistral-nemo-12b-instruct-q4",
"name": "Mistral Nemo 12B Instruct",
"gguf_file": "Mistral-Nemo-Instruct-2407.Q4_K_M.gguf",
"size_mb": 7477,
"vram_required_gb": 12,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Quality",
"description": "Mistral test model",
"llm_model_name": "mistral-nemo-instruct-2407",
"app_compatibility": {
"hermes_talk": {
"status": "unsupported_until_revalidated",
"reason": "Mistral Talk probe failed on Apple llama-server",
"gpuBackendScope": ["apple"],
"llmBackendScope": ["llama-server"],
},
},
}
(install_dir / ".env").write_text("GPU_BACKEND=apple\nLLM_BACKEND=llama-server\n", encoding="utf-8")
apple_payload = build_models_payload(
_gpu(name="Apple M5 Max", total_mb=131072, backend="apple"),
None,
0,
install_dir,
data_dir,
catalog=[model],
evidence=[],
)
(install_dir / ".env").write_text("GPU_BACKEND=amd\nLLM_BACKEND=lemonade\n", encoding="utf-8")
lemonade_payload = build_models_payload(
_gpu(name="AMD Strix Halo", total_mb=126976, backend="amd"),
None,
0,
install_dir,
data_dir,
catalog=[model],
evidence=[],
)
assert apple_payload["models"][0]["appCompatibility"]["hermesTalk"]["status"] == (
"unsupported_until_revalidated"
)
assert apple_payload["models"][0]["appCompatibility"]["agentViability"]["status"] == "not_agent_viable"
assert lemonade_payload["models"][0]["appCompatibility"]["hermesTalk"]["status"] == "unknown"
assert lemonade_payload["models"][0]["appCompatibility"]["agentViability"]["status"] == "unknown"
def test_model_payload_applies_host_scoped_app_compatibility_from_install_env(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
model = {
"id": "granite4.1-3b-q4",
"name": "Granite 4.1 3B",
"gguf_file": "granite-4.1-3b-Q4_K_M.gguf",
"size_mb": 2100,
"vram_required_gb": 4,
"context_length": 131072,
"quantization": "Q4_K_M",
"specialty": "Tool Use",
"description": "Granite test model",
"llm_model_name": "granite-4.1-3b",
"app_compatibility": {
"hermes_talk": {
"status": "unsupported_until_revalidated",
"reason": "Granite Talk probe timed out on windows-laptop",
"hostScope": ["windows-laptop"],
},
},
}
(install_dir / ".env").write_text("ODS_FLEET_HOST_ID=windows-laptop\n", encoding="utf-8")
windows_payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[model],
evidence=[],
)
(install_dir / ".env").write_text("ODS_FLEET_HOST_ID=strixy\n", encoding="utf-8")
strixy_payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[model],
evidence=[],
)
assert model_compatibility_runtime_context(install_dir)["hosts"] == ["strixy"]
assert windows_payload["models"][0]["appCompatibility"]["hermesTalk"]["status"] == (
"unsupported_until_revalidated"
)
assert windows_payload["models"][0]["appCompatibility"]["agentViability"]["status"] == "not_agent_viable"
assert strixy_payload["models"][0]["appCompatibility"]["hermesTalk"]["status"] == "unknown"
assert strixy_payload["models"][0]["appCompatibility"]["agentViability"]["status"] == "unknown"
def test_host_scope_ignores_a_machine_name_that_matches_a_fleet_host(monkeypatch, tmp_path):
# A user's machine that happens to share a fleet host's name gets no
# fleet-scoped verdicts; only an explicit identity selects them.
import performance_oracle
install_dir = tmp_path / "ods"
install_dir.mkdir()
(install_dir / ".env").write_text("ODS_DEVICE_NAME=strixy\n", encoding="utf-8")
for key in ("ODS_FLEET_HOST_ID", "ODS_COMPATIBILITY_HOST"):
monkeypatch.delenv(key, raising=False)
monkeypatch.setenv("HOSTNAME", "strixy")
monkeypatch.setenv("COMPUTERNAME", "strixy")
monkeypatch.setattr(performance_oracle.platform, "node", lambda: "strixy")
model = {
"id": "scoped-model",
"app_compatibility": {
"perplexica": {
"status": "unsupported_until_revalidated",
"reason": "Global block",
"scopedOverrides": [{
"status": "verified", "reason": "Fleet proof", "hostScope": ["strixy"],
"expiresAt": "2999-01-01T00:00:00Z",
}],
},
},
}
context = model_compatibility_runtime_context(install_dir)
assert context["hosts"] == [] and context["host"] == ""
assert model_app_compatibility(model, runtime_context=context)["perplexica"]["status"] == (
"unsupported_until_revalidated"
)
(install_dir / ".env").write_text("ODS_COMPATIBILITY_HOST=strixy\n", encoding="utf-8")
explicit = model_compatibility_runtime_context(install_dir)
assert explicit["hosts"] == ["strixy"]
assert model_app_compatibility(model, runtime_context=explicit)["perplexica"]["status"] == "verified"
def test_real_catalog_gemma_perplexica_block_is_global():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["gemma3-4b-it-q4"]
windows_laptop = model_app_compatibility(
model,
runtime_context={"host": "windows-laptop", "hosts": ["windows-laptop"]},
)
strixy = model_app_compatibility(
model,
runtime_context={"host": "strixy", "hosts": ["strixy"]},
)
tower2 = model_app_compatibility(
model,
runtime_context={"host": "tower2", "hosts": ["tower2"]},
)
assert windows_laptop["perplexica"]["status"] == "unsupported_until_revalidated"
assert strixy["perplexica"]["status"] == "unsupported_until_revalidated"
assert tower2["perplexica"]["status"] == "unsupported_until_revalidated"
def test_real_catalog_granite32_perplexica_block_includes_towers_and_mac_after_live_failures():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["granite3.2-2b-instruct-q4"]
windows_laptop = model_app_compatibility(
model,
runtime_context={"host": "windows-laptop", "hosts": ["windows-laptop"]},
)
strix_halo = model_app_compatibility(
model,
runtime_context={"host": "strix-halo", "hosts": ["strix-halo"]},
)
tower2 = model_app_compatibility(
model,
runtime_context={"host": "tower2", "hosts": ["tower2"]},
)
m5_mbp = model_app_compatibility(
model,
runtime_context={"host": "m5-mbp", "hosts": ["m5-mbp"]},
)
tower3 = model_app_compatibility(
model,
runtime_context={"host": "tower3", "hosts": ["tower3"]},
)
tower1 = model_app_compatibility(
model,
runtime_context={"host": "tower1", "hosts": ["tower1"]},
)
mac_mini = model_app_compatibility(
model,
runtime_context={"host": "mac-mini", "hosts": ["mac-mini"]},
)
assert windows_laptop["perplexica"]["status"] == "unknown"
assert strix_halo["perplexica"]["status"] == "unknown"
assert tower2["perplexica"]["status"] == "unsupported_until_revalidated"
assert m5_mbp["perplexica"]["status"] == "unsupported_until_revalidated"
assert tower3["perplexica"]["status"] == "unsupported_until_revalidated"
assert tower1["perplexica"]["status"] == "unsupported_until_revalidated"
assert mac_mini["perplexica"]["status"] == "unsupported_until_revalidated"
assert "Tower3" in tower3["perplexica"]["reason"]
assert "Tower1" in tower1["perplexica"]["reason"]
assert "M4 Mac Mini" in mac_mini["perplexica"]["reason"]
def test_real_catalog_smollm3_perplexica_block_is_global():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["smollm3-3b-q4"]
lemonade = model_app_compatibility(
model,
runtime_context={
"host": "strix-halo",
"hosts": ["strix-halo"],
"llmBackend": "lemonade",
"runtime": "lemonade",
},
)
llama_server = model_app_compatibility(
model,
runtime_context={
"host": "tower2",
"hosts": ["tower2"],
"llmBackend": "llama-server",
"runtime": "llama-server",
},
)
assert lemonade["perplexica"]["status"] == "unsupported_until_revalidated"
assert lemonade["agentViability"]["status"] == "agent_viable"
assert llama_server["perplexica"]["status"] == "unsupported_until_revalidated"
assert llama_server["agentViability"]["status"] == "agent_viable"
def test_real_catalog_granite41_opencode_block_is_strix_halo_and_spark_scoped():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["granite4.1-3b-q4"]
strix_halo = model_app_compatibility(
model,
runtime_context={"host": "strix-halo", "hosts": ["strix-halo"]},
)
spark = model_app_compatibility(
model,
runtime_context={"host": "spark", "hosts": ["spark"]},
)
tower2 = model_app_compatibility(
model,
runtime_context={"host": "tower2", "hosts": ["tower2"]},
)
assert strix_halo["opencode"]["status"] == "unsupported_until_revalidated"
assert spark["opencode"]["status"] == "unsupported_until_revalidated"
assert tower2["opencode"]["status"] == "unknown"
def test_real_catalog_qwen25_coder_15b_blocks_include_spark_and_m5():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["qwen2.5-coder-1.5b-128k-q4"]
spark = model_app_compatibility(
model,
runtime_context={"host": "spark", "hosts": ["spark"]},
)
m5_mbp = model_app_compatibility(
model,
runtime_context={"host": "m5-mbp", "hosts": ["m5-mbp"]},
)
assert spark["opencode"]["status"] == "unsupported_until_revalidated"
assert spark["agentViability"]["status"] == "not_agent_viable"
assert m5_mbp["opencode"]["status"] == "unknown"
assert m5_mbp["perplexica"]["status"] == "unsupported_until_revalidated"
assert m5_mbp["agentViability"]["status"] == "not_agent_viable"
def test_real_catalog_qwen3_4b_instruct_windows_revalidation_is_verified():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["qwen3-4b-instruct-2507-q4"]
windows_laptop = model_app_compatibility(
model,
runtime_context={"host": "windows-laptop", "hosts": ["windows-laptop"]},
)
strix_halo = model_app_compatibility(
model,
runtime_context={"host": "strix-halo", "hosts": ["strix-halo"]},
)
tower2 = model_app_compatibility(
model,
runtime_context={"host": "tower2", "hosts": ["tower2"]},
)
assert windows_laptop["openaiChat"]["status"] == "verified"
assert windows_laptop["hermesTalk"]["status"] == "verified"
assert windows_laptop["perplexica"]["status"] == "verified"
assert windows_laptop["agentViability"]["status"] == "verified"
assert "Q4-KV profile" in windows_laptop["agentViability"]["reason"]
assert strix_halo["agentViability"]["status"] == "unknown"
assert tower2["agentViability"]["status"] == "unknown"
def test_real_catalog_qwen35_4b_windows_revalidation_is_verified():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["qwen3.5-4b-q4"]
windows_laptop = model_app_compatibility(
model,
runtime_context={"host": "windows-laptop", "hosts": ["windows-laptop"]},
)
strix_halo = model_app_compatibility(
model,
runtime_context={"host": "strix-halo", "hosts": ["strix-halo"]},
)
assert windows_laptop["openaiChat"]["status"] == "verified"
assert windows_laptop["hermesTalk"]["status"] == "verified"
assert windows_laptop["perplexica"]["status"] == "verified"
assert windows_laptop["agentViability"]["status"] == "verified"
assert strix_halo["hermesTalk"]["status"] == "unknown"
assert strix_halo["agentViability"]["status"] == "unknown"
def test_real_catalog_qwen3_4b_128k_talk_block_is_m5_and_windows_scoped():
by_id = {model["id"]: model for model in _official_model_catalog()}
model = by_id["qwen3-4b-128k-q4"]
m5_mbp = model_app_compatibility(
model,
runtime_context={"host": "m5-mbp", "hosts": ["m5-mbp"]},
)
windows_laptop = model_app_compatibility(
model,
runtime_context={"host": "windows-laptop", "hosts": ["windows-laptop"]},
)
assert m5_mbp["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert m5_mbp["agentViability"]["status"] == "not_agent_viable"
assert windows_laptop["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert windows_laptop["agentViability"]["status"] == "not_agent_viable"
def test_measured_local_too_slow_blocks_agent_compatibility(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
model = {
"id": "phi4-mini-q4",
"name": "Phi-4 Mini",
"gguf_file": "Phi-4-mini-instruct-Q4_K_M.gguf",
"size_mb": 2490,
"vram_required_gb": 4,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Balanced",
"description": "Compact 128K model.",
"llm_model_name": "phi-4-mini",
}
record_model_performance(
"phi-4-mini",
"NVIDIA GeForce RTX 4060",
"nvidia",
0.5,
model_id="phi4-mini-q4",
gguf="Phi-4-mini-instruct-Q4_K_M.gguf",
context_length=128000,
vram_total_mb=8192,
)
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[model],
evidence=[],
)
compatibility = payload["models"][0]["appCompatibility"]
assert payload["models"][0]["performance"]["source"] == "measured_local"
assert payload["models"][0]["tokensPerSec"] == 0.5
assert compatibility["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert compatibility["agentViability"]["status"] == "not_agent_viable"
assert "0.5 tok/s" in compatibility["agentViability"]["reason"]
assert compatibility["hermesTalk"]["userMessage"] == (
"This model is too slow on this machine for ODS Talk "
"(0.5 tokens/sec measured, 2+ needed). Switch to a smaller or faster model to use ODS Talk."
)
assert compatibility["agentViability"]["userMessage"] == (
"Too slow on this machine for agent tasks (0.5 tokens/sec measured, 2+ needed)."
)
assert "agent-required" not in compatibility["pixelAgent"]["userMessage"]
def test_published_exact_too_slow_blocks_agent_compatibility(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
model = {
"id": "phi4-mini-q4",
"name": "Phi-4 Mini",
"gguf_file": "Phi-4-mini-instruct-Q4_K_M.gguf",
"size_mb": 2490,
"vram_required_gb": 4,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Balanced",
"description": "Compact 128K model.",
"llm_model_name": "phi-4-mini",
}
evidence = [{
"model_id": "phi4-mini-q4",
"model_names": ["phi-4-mini", "Phi-4-mini-instruct-Q4_K_M.gguf"],
"quantization": "Q4_K_M",
"backend": "nvidia",
"gpu_name": "NVIDIA GeForce RTX 4060",
"vram_gb": 8,
"context_length": 128000,
"runtime": "llama-server",
"tokens_per_second": 0.5,
}]
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=[model],
evidence=evidence,
)
compatibility = payload["models"][0]["appCompatibility"]
assert payload["models"][0]["performance"]["source"] == "published_exact"
assert compatibility["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert compatibility["agentViability"]["status"] == "not_agent_viable"
def test_bundled_windows_laptop_phi_evidence_blocks_agent_compatibility(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
model = {
"id": "phi4-mini-q4",
"name": "Phi-4 Mini",
"gguf_file": "Phi-4-mini-instruct-Q4_K_M.gguf",
"size_mb": 2490,
"vram_required_gb": 4,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Balanced",
"description": "Compact 128K model.",
"llm_model_name": "phi-4-mini",
}
payload = build_models_payload(
_gpu(name="NVIDIA GeForce RTX 5070 Laptop GPU", total_mb=8188),
None,
0,
install_dir,
data_dir,
catalog=[model],
evidence=load_evidence(),
)
compatibility = payload["models"][0]["appCompatibility"]
assert payload["models"][0]["performance"]["source"] == "published_exact"
assert payload["models"][0]["tokensPerSec"] == 0.5
assert compatibility["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert compatibility["agentViability"]["status"] == "not_agent_viable"
def test_real_catalog_has_six_windows_8gb_release_swap_candidates(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
(install_dir / ".env").write_text(
"ODS_FLEET_HOST_ID=windows-laptop\nSYSTEM_RAM_GB=31\n",
encoding="utf-8",
)
catalog = _official_model_catalog()
payload = build_models_payload(
_gpu(name="NVIDIA GeForce RTX 5070 Laptop GPU", total_mb=8188),
"qwen3.5-9b",
0,
install_dir,
data_dir,
catalog=catalog,
evidence=load_evidence(),
)
candidates = [
model for model in payload["models"]
if model["id"] != "qwen3.5-9b-q4"
and model["status"] in {"available", "downloaded"}
and model["fitsVram"] is not False
and model["contextLength"] >= 64000
and all(
not _compatibility_blocks_release_coverage(entry)
for key, entry in model["appCompatibility"].items()
if key != "pixelAgent"
)
]
candidate_ids = {model["id"] for model in candidates}
by_id = {model["id"]: model for model in candidates}
all_by_id = {model["id"]: model for model in payload["models"]}
assert len(candidates) >= 6
assert {
"qwen2.5-coder-3b-128k-q4",
"granite4.0-h-micro-q4",
"granite4.0-h-tiny-q4",
"granite4.0-h-1b-q4",
"qwen3-4b-instruct-2507-q4",
"qwen3.5-4b-q4",
}.issubset(candidate_ids)
assert "qwen3.5-4b-q4" in candidate_ids
assert by_id["qwen3.5-4b-q4"]["contextLength"] >= 64000
assert by_id["qwen3.5-4b-q4"]["appCompatibility"]["agentViability"]["status"] == (
"verified"
)
assert by_id["qwen3.5-4b-q4"]["appCompatibility"]["pixelAgent"]["status"] == (
"not_agent_viable"
)
assert all_by_id["smollm3-3b-q4"]["contextLength"] == 65536
assert all_by_id["qwen3-4b-128k-q4"]["contextLength"] == 131072
assert by_id["qwen2.5-coder-3b-128k-q4"]["contextLength"] == 128000
assert by_id["qwen2.5-coder-3b-128k-q4"]["appCompatibility"]["hermesTalk"]["status"] == (
"unknown"
)
assert by_id["qwen2.5-coder-3b-128k-q4"]["appCompatibility"]["agentViability"]["status"] == (
"verified"
)
assert by_id["qwen2.5-coder-3b-128k-q4"]["appCompatibility"]["pixelAgent"]["status"] == (
"not_agent_viable"
)
assert "qwen3-4b-instruct-2507-q4" in candidate_ids
assert by_id["qwen3-4b-instruct-2507-q4"]["contextLength"] >= 64000
assert by_id["qwen3-4b-instruct-2507-q4"]["appCompatibility"]["agentViability"]["status"] == (
"verified"
)
assert "phi3.5-mini-q4" not in candidate_ids
assert (
all_by_id["phi3.5-mini-q4"]["appCompatibility"]["openaiChat"]["status"]
== "unsupported_until_revalidated"
)
assert all_by_id["falcon-h1-1.5b-instruct-q4"]["appCompatibility"]["hermesTalk"]["status"] == (
"unsupported_until_revalidated"
)
assert all_by_id["falcon-h1-1.5b-instruct-q4"]["appCompatibility"]["agentViability"]["status"] == (
"not_agent_viable"
)
assert all_by_id["falcon-h1-3b-instruct-q4"]["appCompatibility"]["hermesTalk"]["status"] == (
"unsupported_until_revalidated"
)
assert all_by_id["falcon-h1-3b-instruct-q4"]["appCompatibility"]["agentViability"]["status"] == (
"not_agent_viable"
)
assert (
all_by_id["qwen2.5-coder-1.5b-128k-q4"]["appCompatibility"]["perplexica"]["status"]
== "unsupported_until_revalidated"
)
assert "qwen2.5-coder-1.5b-128k-q4" not in candidate_ids
assert all_by_id["phi3-mini-128k-q4"]["appCompatibility"]["hermesTalk"]["status"] == "unknown"
assert all_by_id["phi3-mini-128k-q4"]["appCompatibility"]["perplexica"]["status"] == (
"unsupported_until_revalidated"
)
assert all_by_id["granite4.1-3b-q4"]["appCompatibility"]["hermesTalk"]["status"] == (
"unsupported_until_revalidated"
)
assert all_by_id["granite3.1-2b-instruct-q4"]["appCompatibility"]["perplexica"]["status"] == (
"unsupported_until_revalidated"
)
assert all_by_id["granite4.0-h-1b-q4"]["appCompatibility"]["perplexica"]["status"] == "unknown"
assert "granite3.1-2b-instruct-q4" not in candidate_ids
assert "granite4.0-h-1b-q4" in candidate_ids
# The Strixy-only positive evidence must not override this Windows host's
# independently measured 0.5 tok/s performance block.
assert "phi4-mini-q4" not in candidate_ids
assert "gemma3-4b-it-q4" not in candidate_ids
assert "falcon-h1-1.5b-instruct-q4" not in candidate_ids
assert "falcon-h1-3b-instruct-q4" not in candidate_ids
assert "granite4.1-3b-q4" not in candidate_ids
assert "granite4.0-h-350m-q4" not in candidate_ids
assert "granite4.0-1b-q4" not in candidate_ids
assert "phi3-mini-128k-q4" not in candidate_ids
assert "granite3.3-8b-instruct-q4" not in candidate_ids
assert "smollm3-3b-q4" not in candidate_ids
assert (
all_by_id["smollm3-3b-q4"]["appCompatibility"]["perplexica"]["status"]
== "unsupported_until_revalidated"
)
assert "qwen3-4b-128k-q4" not in candidate_ids
assert (
all_by_id["qwen3-4b-128k-q4"]["appCompatibility"]["hermesTalk"]["status"]
== "unsupported_until_revalidated"
)
assert "qwen2.5-3b-instruct-q4" not in candidate_ids
assert "qwen3-4b-q4" not in candidate_ids
assert "qwen3-1.7b-q4" not in candidate_ids
def test_real_catalog_scopes_qwen25_coder_3b_host_failures():
catalog = _official_model_catalog()
model = next(model for model in catalog if model["id"] == "qwen2.5-coder-3b-128k-q4")
tower = model_app_compatibility(model, runtime_context={"hosts": ["tower2"]})
halo = model_app_compatibility(model, runtime_context={"hosts": ["strix-halo"]})
spark = model_app_compatibility(model, runtime_context={"hosts": ["spark"]})
m5_mbp = model_app_compatibility(model, runtime_context={"hosts": ["m5-mbp"]})
windows = model_app_compatibility(model, runtime_context={"hosts": ["windows-laptop"]})
assert tower["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert tower["agentViability"]["status"] == "unknown"
assert tower["opencode"]["status"] == "unknown"
assert halo["hermesTalk"]["status"] == "unknown"
assert halo["opencode"]["status"] == "unsupported_until_revalidated"
assert halo["agentViability"]["status"] == "unknown"
assert spark["hermesTalk"]["status"] == "unknown"
assert spark["opencode"]["status"] == "unsupported_until_revalidated"
assert spark["agentViability"]["status"] == "unknown"
assert m5_mbp["hermesTalk"]["status"] == "unsupported_until_revalidated"
assert "cycle-005/m5-mbp" in m5_mbp["hermesTalk"]["evidence"]
assert m5_mbp["opencode"]["status"] == "unknown"
assert m5_mbp["agentViability"]["status"] == "unknown"
assert any(_compatibility_blocks_release_coverage(entry) for entry in m5_mbp.values())
assert windows["hermesTalk"]["status"] == "unknown"
assert windows["opencode"]["status"] == "unknown"
assert windows["agentViability"]["status"] == "verified"
assert windows["pixelAgent"]["status"] == "not_agent_viable"
assert not any(
_compatibility_blocks_release_coverage(entry)
for key, entry in windows.items()
if key != "pixelAgent"
)
def test_real_catalog_scopes_granite_h_tiny_pixel_failure_to_proven_hosts():
catalog = _official_model_catalog()
model = next(model for model in catalog if model["id"] == "granite4.0-h-tiny-q4")
tower3 = model_app_compatibility(model, runtime_context={"hosts": ["tower3"]})
tower2 = model_app_compatibility(model, runtime_context={"hosts": ["tower2"]})
windows = model_app_compatibility(
model,
runtime_context={"hosts": ["windows-laptop"]},
)
tower1 = model_app_compatibility(model, runtime_context={"hosts": ["tower1"]})
assert tower3["pixelAgent"]["status"] == "unsupported_until_revalidated"
assert tower2["pixelAgent"]["status"] == "unsupported_until_revalidated"
assert windows["pixelAgent"]["status"] == "unsupported_until_revalidated"
assert "cycle-003/tower2/model-ui.json" in tower2["pixelAgent"]["evidence"]
assert "cycle-005/windows-laptop-wsl-beta/model-ui.json" in (
windows["pixelAgent"]["evidence"]
)
assert tower1["pixelAgent"]["status"] == "unknown"
def test_real_catalog_scopes_granite_h_1b_perplexica_failure_to_proven_hosts():
catalog = _official_model_catalog()
model = next(model for model in catalog if model["id"] == "granite4.0-h-1b-q4")
macbook = model_app_compatibility(model, runtime_context={"hosts": ["m5-mbp"]})
windows_wsl = model_app_compatibility(
model,
runtime_context={"hosts": ["windows-laptop-wsl-beta"]},
)
tower1 = model_app_compatibility(model, runtime_context={"hosts": ["tower1"]})
assert macbook["perplexica"]["status"] == "unsupported_until_revalidated"
assert windows_wsl["perplexica"]["status"] == "unsupported_until_revalidated"
assert "cycle-003/windows-laptop-wsl-beta/model-ui.json" in (
windows_wsl["perplexica"]["evidence"]
)
assert tower1["perplexica"]["status"] == "unknown"
def test_real_catalog_scopes_phi4_talk_pass_and_later_pixel_failure_to_strixy():
catalog = _official_model_catalog()
model = next(model for model in catalog if model["id"] == "phi4-mini-q4")
strixy = model_app_compatibility(model, runtime_context={"hosts": ["strixy"]})
windows = model_app_compatibility(model, runtime_context={"hosts": ["windows-laptop"]})
assert strixy["hermesTalk"]["status"] == "verified"
assert strixy["pixelAgent"]["status"] == "unsupported_until_revalidated"
assert strixy["openWebui"]["status"] == "unsupported_until_revalidated"
assert strixy["agentViability"]["status"] == "unsupported_until_revalidated"
assert "cycle-005/strixy-wsl-beta/model-ui.json" in strixy["hermesTalk"]["evidence"]
assert windows["hermesTalk"]["status"] == "unknown"
assert windows["pixelAgent"]["status"] == "unknown"
assert windows["agentViability"]["status"] == "unknown"
def test_real_catalog_scopes_qwen35_9b_open_webui_revalidation_to_strixy():
catalog = _official_model_catalog()
model = next(model for model in catalog if model["id"] == "qwen3.5-9b-q4")
strixy = model_app_compatibility(model, runtime_context={"hosts": ["strixy"]})
windows = model_app_compatibility(model, runtime_context={"hosts": ["windows-laptop"]})
assert strixy["openWebui"]["status"] == "verified"
assert "cycle-006/strixy-wsl-beta/model-ui.json" in strixy["openWebui"]["evidence"]
assert windows["openWebui"]["status"] == "unknown"
def test_installer_recommended_model_survives_bootstrap_env(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
(install_dir / ".env").write_text(
"LLM_MODEL=qwen3.5-2b\n"
"GGUF_FILE=Qwen3.5-2B-Q4_K_M.gguf\n"
"MODEL_RECOMMENDED_MODEL=qwen3.5-9b\n"
"MODEL_RECOMMENDED_GGUF=Qwen3.5-9B-Q4_K_M.gguf\n"
"MODEL_RECOMMENDED_CONTEXT=65536\n"
"MODEL_RECOMMENDATION_SOURCE=installer_tier_map\n",
encoding="utf-8",
)
catalog = [
{
"id": "qwen3.5-2b-q4",
"name": "Qwen 3.5 2B",
"gguf_file": "Qwen3.5-2B-Q4_K_M.gguf",
"size_mb": 1500,
"vram_required_gb": 3,
"context_length": 8192,
"quantization": "Q4_K_M",
"specialty": "Fast",
"description": "Bootstrap model",
"llm_model_name": "qwen3.5-2b",
},
_model(),
]
payload = build_models_payload(_gpu(), "qwen3.5-2b", 60, install_dir, data_dir, catalog=catalog, evidence=[])
by_id = {model["id"]: model for model in payload["models"]}
assert payload["currentModel"] == "qwen3.5-2b-q4"
assert payload["configuredModel"] == "qwen3.5-9b-q4"
assert payload["hermesMinimumContext"] == 65536
assert payload["hermesTargetContext"] == 131072
assert payload["pixelMinimumContext"] == 16384
assert by_id["qwen3.5-2b-q4"]["status"] == "loaded"
assert by_id["qwen3.5-9b-q4"]["contextLength"] == 65536
assert by_id["qwen3.5-9b-q4"]["recommended"] is True
assert by_id["qwen3.5-9b-q4"]["recommendation"]["source"] == "installer_tier_map"
assert by_id["qwen3.5-9b-q4"]["recommendation"]["contextLength"] == 65536
assert payload["recommendationAlternatives"][0]["id"] == "qwen3.5-9b-q4"
def test_switch_plan_clamps_a_stale_context_to_the_declared_native_context():
"""A switch or restore never asks llama.cpp for more than a declared native
context (it caps the slot there and the context proof fails, #6712).
phi-4 declares 16,384 and Qwen3-30B-A3B 40,960 in the catalog; a stale
CTX_SIZE / MODEL_RECOMMENDED_CONTEXT above that plans at the native value.
"""
catalog = {raw["id"]: normalize_catalog_entry(raw) for raw in _official_model_catalog()}
phi4 = planned_model_context(catalog["phi4-q4"], _gpu(total_mb=24576), 64, preferred_context=65536)
assert phi4["context_length"] == 16384
assert phi4["meets_min_context"] is False
qwen3_30b = planned_model_context(
catalog["qwen3-30b-a3b-q4"], _gpu(total_mb=49140), 128, preferred_context=131072,
)
assert qwen3_30b["context_length"] == 40960
def test_switch_plan_keeps_an_owner_context_when_no_native_context_is_declared():
# Normalization fills max_context_length from context_length for the
# context options when the catalog declares none; that fallback is not a
# native ceiling, so an owner's (or the installer's) larger context stays.
entry = normalize_catalog_entry(_model())
assert entry["max_context_length"] == 32768
assert entry["native_context_declared"] is False
plan = planned_model_context(entry, _gpu(total_mb=16384), 64, preferred_context=65536)
assert plan["context_length"] == 65536
def test_context_options_separate_recommended_context_from_model_limit(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
(install_dir / ".env").write_text(
"LLM_MODEL=qwen3.5-2b\n"
"GGUF_FILE=Qwen3.5-2B-Q4_K_M.gguf\n"
"CTX_SIZE=8192\n"
"MAX_CONTEXT=8192\n",
encoding="utf-8",
)
catalog = [{
"id": "qwen3.5-2b-q4",
"name": "Qwen 3.5 2B",
"gguf_file": "Qwen3.5-2B-Q4_K_M.gguf",
"size_mb": 1500,
"vram_required_gb": 3,
"context_length": 8192,
"max_context_length": 262144,
"quantization": "Q4_K_M",
"specialty": "Fast",
"description": "Bootstrap model",
"llm_model_name": "qwen3.5-2b",
}]
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=catalog,
evidence=[],
)
model = payload["models"][0]
assert model["contextLength"] == 8192
assert model["maxContextLength"] == 262144
assert [option["contextLength"] for option in model["contextOptions"]] == [
8192,
16384,
32768,
65536,
131072,
262144,
]
assert next(
option for option in model["contextOptions"] if option["recommended"]
)["contextLength"] == 8192
assert next(
option for option in model["contextOptions"] if option["fullContext"]
)["contextLength"] == 262144
def test_unknown_import_context_is_not_reported_as_a_declared_limit(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
catalog = [{
"id": "hf-community",
"name": "Community model",
"gguf_file": "community.gguf",
"size_mb": 500,
"vram_required_gb": 1,
"context_length": 8192,
"max_context_length": None,
"context_limit_known": False,
"context_source": "unavailable",
"quantization": "Q4_K_M",
"specialty": "Community GGUF",
"description": "Imported model",
"source": "huggingface",
}]
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
catalog=catalog,
evidence=[],
)
model = payload["models"][0]
assert model["contextLength"] == 8192
assert model["maxContextLength"] is None
assert model["metadata"]["contextLimitKnown"] is False
assert model["metadata"]["contextSource"] == "unavailable"
assert not any(option["fullContext"] for option in model["contextOptions"])
def test_downloaded_gguf_header_replaces_stale_hub_context(
data_dir, tmp_path, monkeypatch,
):
import performance_oracle
install_dir = tmp_path / "ods"
model_path = install_dir / "data" / "models" / "community.gguf"
model_path.parent.mkdir(parents=True)
model_path.write_bytes(b"GGUF")
catalog = [{
"id": "hf-community",
"name": "Community model",
"gguf_file": model_path.name,
"size_mb": 500,
"vram_required_gb": 1,
"context_length": 8192,
"max_context_length": 32768,
"context_limit_known": True,
"context_source": "hub_config",
"quantization": "Q4_K_M",
"specialty": "Community GGUF",
"description": "Imported model",
"source": "huggingface",
}]
monkeypatch.setattr(
performance_oracle,
"inspect_gguf",
lambda _path: {
"exists": True,
"readable": True,
"context_length": 131072,
"quantization": "Q4_K_M",
"block_count": 36,
"attention_head_count_kv": 8,
"embedding_length": 4096,
"attention_head_count": 32,
},
)
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
install_dir / "data",
catalog=catalog,
evidence=[],
)
model = payload["models"][0]
assert model["maxContextLength"] == 131072
assert model["metadata"]["contextLimitKnown"] is True
assert model["metadata"]["contextSource"] == "gguf_file"
assert model["contextOptions"][-1]["contextLength"] == 131072
assert model["contextOptions"][-1]["fullContext"] is True
assert model["estimatedRequired"] == 1.61
assert model["contextOptions"][-1]["estimatedRequired"] == 18.49
def test_configured_model_prefers_env_file_over_stale_process_env(data_dir, tmp_path, monkeypatch):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
(install_dir / ".env").write_text(
"LLM_MODEL=phi-4-mini\n"
"GGUF_FILE=Phi-4-mini-instruct-Q4_K_M.gguf\n",
encoding="utf-8",
)
monkeypatch.setenv("LLM_MODEL", "qwen3.6-35b-a3b")
monkeypatch.setenv("GGUF_FILE", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf")
catalog = [
{
"id": "phi4-mini-q4",
"name": "Phi-4 Mini",
"gguf_file": "Phi-4-mini-instruct-Q4_K_M.gguf",
"size_mb": 2490,
"vram_required_gb": 4,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Balanced",
"description": "Compact 128K model.",
"llm_model_name": "phi-4-mini",
},
{
"id": "qwen3.6-35b-a3b-ud-q4",
"name": "Qwen 3.6 35B A3B",
"gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf",
"size_mb": 21500,
"vram_required_gb": 24,
"context_length": 65536,
"quantization": "UD-Q4_K_M",
"specialty": "Reasoning",
"description": "Large reasoning model.",
"llm_model_name": "qwen3.6-35b-a3b",
},
]
payload = build_models_payload(
_gpu(),
"Phi-4-mini-instruct-Q4_K_M",
0,
install_dir,
data_dir,
catalog=catalog,
evidence=[],
)
by_id = {model["id"]: model for model in payload["models"]}
assert payload["currentModel"] == "phi4-mini-q4"
assert payload["configuredModel"] == "phi4-mini-q4"
assert by_id["phi4-mini-q4"]["configured"] is True
assert by_id["qwen3.6-35b-a3b-ud-q4"]["configured"] is False
def test_pre_download_ranker_prefers_capable_8gb_model_over_bootstrap(data_dir):
catalog = [
{
"id": "qwen3.5-2b-q4",
"name": "Qwen 3.5 2B",
"gguf_file": "Qwen3.5-2B-Q4_K_M.gguf",
"size_mb": 1500,
"vram_required_gb": 3,
"context_length": 8192,
"quantization": "Q4_K_M",
"specialty": "Fast",
"description": "Bootstrap model",
"llm_model_name": "qwen3.5-2b",
},
_model(),
]
ranked = rank_pre_download_models(catalog, _gpu(total_mb=8188), profile="qwen", limit=2)
assert ranked[0]["id"] == "qwen3.5-9b-q4"
def test_pre_download_ranker_accounts_for_long_context_kv_on_4gb_gpu(data_dir, tmp_path):
catalog = [
{
"id": "phi4-mini-q4",
"name": "Phi-4 Mini",
"gguf_file": "Phi-4-mini-instruct-Q4_K_M.gguf",
"size_mb": 2490,
"vram_required_gb": 4,
"context_length": 128000,
"quantization": "Q4_K_M",
"specialty": "Balanced",
"description": "Compact 128K model.",
"llm_model_name": "phi-4-mini",
},
{
"id": "qwen3.5-2b-q4",
"name": "Qwen 3.5 2B",
"family": "qwen",
"gguf_file": "Qwen3.5-2B-Q4_K_M.gguf",
"size_mb": 1500,
"vram_required_gb": 3,
"context_length": 8192,
"quantization": "Q4_K_M",
"specialty": "Fast",
"description": "Bootstrap model",
"llm_model_name": "qwen3.5-2b",
},
]
gpu = _gpu(total_mb=4096)
ranked = rank_pre_download_models(catalog, gpu, profile="qwen", limit=2)
assert ranked[0]["id"] == "qwen3.5-2b-q4"
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(gpu, None, 0, install_dir, data_dir, catalog=catalog, evidence=[])
by_id = {model["id"]: model for model in payload["models"]}
assert by_id["phi4-mini-q4"]["fitsVram"] is False
assert by_id["phi4-mini-q4"]["estimatedRequired"] > by_id["phi4-mini-q4"]["vramRequired"]
def test_qwen35_2b_is_the_4gb_recommendation_despite_fleet_failures(
data_dir,
tmp_path,
monkeypatch,
):
monkeypatch.setenv("ODS_FLEET_HOST_ID", "tower2")
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(total_mb=4096),
None,
0,
install_dir,
data_dir,
catalog=_official_model_catalog(),
evidence=[],
)
model = next(item for item in payload["models"] if item["id"] == "qwen3.5-2b-q4")
assert model["contextLength"] == 65536
assert model["maxContextLength"] == 262144
assert model["vramRequired"] == 3
assert model["estimatedRequired"] <= 4
assert model["fitsVram"] is True
# Nothing else installable fits a 4 GB card at the 64K Hermes floor
# (phi-4-mini only reaches 8K there), so the 2B is recommended; its
# fleet verdicts still say where it falls short.
assert model["recommended"] is True
compatibility = model["appCompatibility"]
assert compatibility["hermesTalk"]["status"] == "verified"
assert compatibility["openaiChat"]["status"] == "unsupported_until_revalidated"
assert compatibility["perplexica"]["status"] == "unsupported_until_revalidated"
assert compatibility["agentViability"]["status"] == "not_agent_viable"
def test_jamba_reasoning_3b_catalog_profile_fits_4gb_at_agent_context(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(total_mb=4096),
None,
0,
install_dir,
data_dir,
catalog=_official_model_catalog(),
evidence=[],
)
model = next(item for item in payload["models"] if item["id"] == "jamba-reasoning-3b-q4")
assert model["contextLength"] == 65536
assert model["maxContextLength"] == 262144
assert model["vramRequired"] == 3
assert model["estimatedRequired"] <= 4
assert model["fitsVram"] is True
assert model["recommended"] is False
def test_pre_download_ranker_does_not_assume_large_gpu_without_hardware_info(data_dir):
catalog = [
_model(),
{
"id": "qwen3.6-35b-a3b-ud-q4",
"name": "Qwen 3.6 35B-A3B",
"family": "qwen",
"gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf",
"size_mb": 21110,
"vram_required_gb": 24,
"context_length": 131072,
"quantization": "UD-Q4_K_M",
"specialty": "Quality",
"description": "Large MoE model",
"llm_model_name": "qwen3.6-35b-a3b",
},
]
ranked = rank_pre_download_models(catalog, None, profile="qwen", limit=2)
assert ranked == []
def test_windows_amd_host_runtime_uses_install_ram_when_gpu_probe_is_unavailable(data_dir, tmp_path):
install_dir = tmp_path / "ods"
install_dir.mkdir(parents=True)
models_dir = data_dir / "models"
models_dir.mkdir(parents=True)
(models_dir / "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf").write_text("placeholder", encoding="utf-8")
(install_dir / ".env").write_text(
"GPU_BACKEND=amd\n"
"LLM_BACKEND=llama-server\n"
"AMD_INFERENCE_RUNTIME=llama-server\n"
"AMD_INFERENCE_LOCATION=host\n"
"SYSTEM_RAM_GB=128\n"
"MODEL_RECOMMENDATION_POLICY=context-aware-curated-fit-v2+unified-memory-coder-next-a3b-v1\n",
encoding="utf-8",
)
catalog = [{
"id": "qwen3.6-35b-a3b-ud-q4",
"name": "Qwen 3.6 35B-A3B",
"family": "qwen",
"gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf",
"size_mb": 21110,
"vram_required_gb": 24,
"context_length": 131072,
"quantization": "UD-Q4_K_M",
"specialty": "Quality",
"description": "Large MoE model",
"llm_model_name": "qwen3.6-35b-a3b",
"runtime_profiles": [{
"id": "amd-strix-halo-unified",
"label": "AMD Strix Halo unified-memory profile",
"backend": "amd",
"memory_type": "unified",
"vram_min_gb": 90,
"system_ram_min_gb": 64,
"estimated_required_gb": 44,
"context_length": 131072,
"fit_label": "Host AMD unified-memory fit",
}],
}]
payload = build_models_payload(None, None, 0, install_dir, data_dir, catalog=catalog, evidence=[])
model = payload["models"][0]
assert payload["gpu"]["vramTotal"] == 128
assert payload["gpu"]["vramFree"] == 128
assert model["status"] == "downloaded"
assert model["fitsVram"] is True
assert model["runtimeProfile"]["id"] == "amd-strix-halo-unified"
def test_pre_download_ranker_honors_gemma_profile(data_dir):
catalog = [
_model(),
{
"id": "gemma4-e4b-q4",
"name": "Gemma 4 E4B",
"family": "gemma4",
"gguf_file": "gemma-4-E4B-it-Q4_K_M.gguf",
"size_mb": 5340,
"vram_required_gb": 8,
"context_length": 32768,
"quantization": "Q4_K_M",
"specialty": "General",
"description": "Gemma profile model",
"llm_model_name": "gemma-4-e4b-it",
},
]
ranked = rank_pre_download_models(catalog, _gpu(total_mb=8188), profile="gemma4", limit=2)
assert ranked[0]["id"] == "gemma4-e4b-q4"
def test_pre_download_ranker_allows_8gb_nvidia_runtime_profile(monkeypatch):
monkeypatch.setattr("performance_oracle._system_ram_gb", lambda: 31)
monkeypatch.setattr("performance_oracle.platform.machine", lambda: "x86_64")
catalog = [
_model(),
{
"id": "qwen3.6-35b-a3b-ud-q4",
"name": "Qwen 3.6 35B-A3B",
"family": "qwen",
"gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf",
"size_mb": 21110,
"vram_required_gb": 24,
"context_length": 131072,
"quantization": "UD-Q4_K_M",
"specialty": "Quality",
"description": "Large MoE model",
"llm_model_name": "qwen3.6-35b-a3b",
"runtime_profiles": [{
"id": "nvidia-8gb-qwen36-35b-a3b-turboquant",
"label": "Advanced 8GB NVIDIA TurboQuant MoE offload",
"backend": "nvidia",
"host_arch": ["amd64"],
"memory_type": "discrete",
"vram_min_gb": 7.5,
"vram_max_gb": 12.5,
"system_ram_min_gb": 31,
"estimated_required_gb": 8,
"context_length": 65536,
"fit_label": "Advanced 8GB TurboQuant fit",
"env": {"LLAMA_ARG_N_CPU_MOE": "30"},
}],
},
]
ranked = rank_pre_download_models(catalog, _gpu(total_mb=8188), profile="qwen", limit=2)
assert ranked[0]["id"] == "qwen3.6-35b-a3b-ud-q4"
assert ranked[0]["_runtime_profile"]["id"] == "nvidia-8gb-qwen36-35b-a3b-turboquant"
def test_pre_download_ranker_excludes_ram_ineligible_hardware_profile(
monkeypatch,
data_dir,
tmp_path,
):
monkeypatch.setattr("performance_oracle.platform.machine", lambda: "x86_64")
small = {
"id": "plain-small",
"name": "Plain Small",
"family": "qwen",
"gguf_file": "plain-small.gguf",
"size_mb": 1024,
"vram_required_gb": 2,
"context_length": 8192,
"quantization": "Q4_K_M",
"specialty": "Fast",
"description": "Unprofiled fallback",
"llm_model_name": "plain-small",
}
profiled = {
**_model(),
"runtime_profiles": [{
"id": "nvidia-8gb-profile",
"backend": "nvidia",
"host_arch": ["amd64"],
"memory_type": "discrete",
"vram_min_gb": 7.5,
"vram_max_gb": 8.5,
"system_ram_min_gb": 15,
"estimated_required_gb": 8,
"context_length": 65536,
}],
}
constrained = rank_pre_download_models(
[profiled, small],
_gpu(total_mb=8188),
profile="qwen",
limit=2,
system_ram_gb=13,
)
eligible = rank_pre_download_models(
[profiled, small],
_gpu(total_mb=8188),
profile="qwen",
limit=2,
system_ram_gb=15,
)
assert [model["id"] for model in constrained] == ["plain-small"]
assert eligible[0]["id"] == profiled["id"]
assert eligible[0]["_runtime_profile"]["id"] == "nvidia-8gb-profile"
install_dir = tmp_path / "ods"
install_dir.mkdir()
(install_dir / ".env").write_text("SYSTEM_RAM_GB=13\n", encoding="utf-8")
payload = build_models_payload(
_gpu(total_mb=8188),
None,
0,
install_dir,
data_dir,
catalog=[profiled, small],
evidence=[],
)
cards = {model["id"]: model for model in payload["models"]}
assert cards[profiled["id"]]["fitsVram"] is False
assert cards[profiled["id"]]["runtimeProfile"] is None
assert cards["plain-small"]["recommended"] is True
def test_measured_local_from_live_loaded_model(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(),
"qwen3.5-9b",
41.8,
install_dir,
data_dir,
context_length=32768,
catalog=[_model()],
evidence=[],
)
loaded = payload["models"][0]
assert payload["currentModel"] == "qwen3.5-9b-q4"
assert loaded["status"] == "loaded"
assert loaded["performance"]["source"] == "measured_local"
assert loaded["tokensPerSec"] == 41.8
def test_implausible_live_counter_is_not_emitted_as_measured_speed(data_dir, tmp_path):
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(),
"qwen3.5-9b",
1_000_000,
install_dir,
data_dir,
context_length=32768,
catalog=[_model()],
evidence=[],
)
loaded = payload["models"][0]
assert loaded["status"] == "loaded"
assert loaded["performance"]["source"] == "benchmark_required"
assert loaded["tokensPerSec"] is None
def test_predicted_calibrated_requires_local_sample(data_dir, tmp_path):
record_model_performance(
"qwen3.5-4b",
"NVIDIA GeForce RTX 4060",
"nvidia",
80.0,
model_id="qwen3.5-4b-q4",
gguf="Qwen3.5-4B-Q4_K_M.gguf",
context_length=16384,
decode_read_mb=2870,
vram_total_mb=8192,
)
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
context_length=32768,
catalog=[_model()],
evidence=[],
)
perf = payload["models"][0]["performance"]
assert perf["source"] == "predicted_calibrated"
assert perf["tokensPerSec"] is not None
assert perf["confidence"] == "low"
def test_polluted_history_is_ignored_in_favor_of_a_valid_exact_alias(
data_dir, tmp_path, monkeypatch,
):
import performance_oracle
samples = iter([
{"tokens_per_second": 1_000_000, "sample_count": 336},
{"tokens_per_second": 240.5, "sample_count": 1},
])
monkeypatch.setattr(
performance_oracle,
"get_recorded_model_performance",
lambda *args, **kwargs: next(samples, None),
)
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
payload = build_models_payload(
_gpu(),
None,
0,
install_dir,
data_dir,
context_length=32768,
catalog=[_model()],
evidence=[],
)
assert payload["models"][0]["performance"]["source"] == "measured_local"
assert payload["models"][0]["tokensPerSec"] == 240.5
def test_official_catalog_families_expose_their_real_hugging_face_identity():
cases = {
"Qwen 3.5 2B": ("Qwen", "Qwen"),
"Phi-4 Mini": ("Microsoft", "microsoft"),
"Granite 3.3 2B": ("IBM Granite", "ibm-granite"),
"SmolLM3 3B": ("Hugging Face", "HuggingFaceTB"),
"Gemma 3 4B": ("Google", "google"),
"Falcon H1 7B": ("Technology Innovation Institute", "tiiuae"),
"Ministral 3B": ("Mistral AI", "mistralai"),
"Llama 3.2 3B": ("Meta", "meta-llama"),
"DeepSeek R1 7B": ("DeepSeek", "deepseek-ai"),
}
for name, (publisher, author) in cases.items():
assert model_publisher({"name": name}) == {
"name": publisher,
"huggingFaceAuthor": author,
}
def test_publisher_identity_does_not_match_family_substrings_inside_other_words():
assert model_publisher({"name": "Dolphin 2.9 Mixtral 8x7B"}) == {
"name": "Mistral AI",
"huggingFaceAuthor": "mistralai",
}
assert model_publisher({"name": "OpenPhind Code Model"}) is None
def test_calibrated_prediction_above_single_request_ceiling_requires_benchmark(data_dir):
record_model_performance(
"calibration-model",
"NVIDIA GeForce RTX 4060",
"nvidia",
5_000,
decode_read_mb=10_000,
vram_total_mb=8192,
)
performance = evaluate_performance(
{**_model(), "decode_read_mb": 1},
_gpu(),
{"quantization": "Q4_K_M", "readable": False},
False,
0,
32768,
{},
[],
True,
)
assert performance["source"] == "benchmark_required"
def test_published_exact_requires_matching_signature(data_dir):
evidence = [{
"model_id": "qwen3.5-9b-q4",
"model_names": ["qwen3.5-9b", "Qwen3.5-9B-Q4_K_M.gguf"],
"quantization": "Q4_K_M",
"backend": "nvidia",
"gpu_name": "NVIDIA GeForce RTX 4060",
"vram_gb": 8,
"context_length": 32768,
"tokens_per_second": 44.2,
"source_url": "https://example.test/bench",
}]
perf = evaluate_performance(
_model(),
_gpu(),
{"quantization": "Q4_K_M", "readable": False},
False,
0,
32768,
{},
evidence,
True,
)
assert perf["source"] == "published_exact"
assert perf["tokensPerSec"] == 44.2
assert perf["sourceUrl"] == "https://example.test/bench"
def test_published_exact_matches_gguf_stem_identity(data_dir):
evidence = [{
"model_id": "Qwen3.5-9B-Q4_K_M",
"model_names": [],
"quantization": "Q4_K_M",
"backend": "nvidia",
"gpu_name": "NVIDIA GeForce RTX 4060",
"vram_gb": 8,
"context_length": 32768,
"tokens_per_second": 43.7,
"source_url": "https://example.test/stem-bench",
}]
perf = evaluate_performance(
_model(),
_gpu(),
{"quantization": "Q4_K_M", "readable": False},
False,
0,
32768,
{},
evidence,
True,
)
assert perf["source"] == "published_exact"
assert perf["tokensPerSec"] == 43.7
assert perf["sourceUrl"] == "https://example.test/stem-bench"
def _selection_envelopes():
fixture = Path(__file__).resolve().parents[4] / "tests" / "fixtures" / "model-selection-envelopes.json"
return [
envelope for envelope in json.loads(fixture.read_text(encoding="utf-8"))["envelopes"]
if envelope["ceiling"] == 0
]
def _installer_selector():
import importlib.util
path = Path(__file__).resolve().parents[4] / "scripts" / "select-model.py"
spec = importlib.util.spec_from_file_location("ods_select_model_oracle_parity", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_dashboard_ranker_matches_the_installer_on_every_envelope(monkeypatch):
"""The dashboard recommends what scripts/select-model.py ranks first.
Envelopes with a tier size ceiling are installer-only (the dashboard has
none), and the installer's architecture substitutions (Spark, Strix Halo,
unified-memory coder-next) are applied after this shared ranking.
"""
import performance_oracle as oracle
selector = _installer_selector()
installer_catalog = selector.load_catalog(
Path(__file__).resolve().parents[4] / "config" / "model-library.json"
)
dashboard_catalog = [
entry for entry in (normalize_catalog_entry(raw) for raw in _official_model_catalog())
if entry is not None and str(entry.get("catalog_source") or "ods") in {"ods", "curated"}
]
mismatches = []
for envelope in _selection_envelopes():
monkeypatch.setattr(oracle.platform, "machine", lambda arch=envelope["host_arch"]: arch)
capacity, _ = selector.usable_memory_gb(
envelope["backend"], envelope["memory_type"], envelope["vram_mb"], envelope["ram_gb"]
)
installer = selector.rank_models(
installer_catalog, capacity, "qwen", True, envelope["backend"],
envelope["memory_type"], envelope["vram_mb"], envelope["ram_gb"],
envelope["host_arch"], min_context=65536,
)
gpu = GPUInfo(
name="test", memory_used_mb=0,
memory_total_mb=envelope["vram_mb"] or envelope["ram_gb"] * 1024,
memory_percent=0, utilization_percent=0, temperature_c=30,
gpu_backend=envelope["backend"], memory_type=envelope["memory_type"],
)
if envelope["backend"] == "cpu":
gpu = GPUInfo(
name="cpu", memory_used_mb=0, memory_total_mb=0, memory_percent=0,
utilization_percent=0, temperature_c=30, gpu_backend="cpu",
)
dashboard = rank_pre_download_models(
dashboard_catalog, gpu, "qwen", True, limit=1, system_ram_gb=envelope["ram_gb"],
)
got = [
(model["id"], model["context_length"], (model.get("_runtime_profile") or {}).get("id"))
for model in dashboard[:1]
]
want = [
(model["id"], model["context_length"], (model.get("_runtime_profile") or {}).get("id"))
for model in installer[:1]
]
if got != want:
mismatches.append((envelope["id"], want, got))
assert not mismatches, mismatches
def test_talk_verdict_follows_the_context_the_model_is_served_at():
model = {
"id": "qwen3.5-27b-q4", "context_length": 65536, "max_context_length": 262144,
"app_compatibility": {"hermes_talk": {"status": "verified"}},
}
assert model_app_compatibility(model, context_length=65536)["hermesTalk"]["status"] == "verified"
served_low = model_app_compatibility(model, context_length=32768)["hermesTalk"]
assert served_low["status"] == "unsupported"
assert served_low["code"] == "context_below_hermes_minimum"
assert "32K" in served_low["userMessage"] and "64K" in served_low["userMessage"]
# Without a known context the catalog verdict stands.
assert model_app_compatibility(model)["hermesTalk"]["status"] == "verified"
def test_a_failed_tool_check_adds_a_talk_advisory_not_a_block():
# Any-model WP4.4 (owner decision: advisory only, no lockout).
model = {
"id": "import", "context_length": 65536, "max_context_length": 131072,
"app_compatibility": {"hermes_talk": {"status": "verified"}},
}
measured = model_app_compatibility(model, context_length=65536, profile_tools=False)
assert measured["hermesTalk"]["status"] == "verified"
assert measured["hermesTalkAdvisory"] == {"code": "tools-unavailable"}
unchanged = model_app_compatibility(model, context_length=65536)
assert "hermesTalkAdvisory" not in unchanged
assert unchanged == {key: value for key, value in measured.items() if key != "hermesTalkAdvisory"}
assert "hermesTalkAdvisory" not in model_app_compatibility(model, context_length=65536, profile_tools=True)
def test_a_talk_block_keeps_priority_over_the_tools_advisory():
model = {"id": "import", "context_length": 65536, "max_context_length": 131072}
blocked = model_app_compatibility(model, context_length=32768, profile_tools=False)
assert blocked["hermesTalk"]["code"] == "context_below_hermes_minimum"
assert "hermesTalkAdvisory" not in blocked
def test_talk_verdict_names_a_native_context_limit():
native = model_app_compatibility({"id": "phi4-q4", "context_length": 16384, "max_context_length": 16384})
assert native["hermesTalk"]["status"] == "unsupported"
assert "supports only 16K" in native["hermesTalk"]["userMessage"]
# An unknown limit (a GGUF whose header was unreadable) is not guessed.
unknown = model_app_compatibility({
"id": "import", "context_length": 8192, "max_context_length": 8192, "context_limit_known": False,
})
assert unknown["hermesTalk"]["status"] == "unknown"
def test_existing_blocking_verdict_keeps_its_own_copy():
model = {
"id": "granite", "context_length": 131072, "max_context_length": 131072,
"app_compatibility": {"hermes_talk": {
"status": "unsupported_until_revalidated", "userNote": "Granite can't keep up with Talk yet.",
}},
}
talk = model_app_compatibility(model, context_length=32768)["hermesTalk"]
assert talk["status"] == "unsupported_until_revalidated"
assert talk["userMessage"] == "Granite can't keep up with Talk yet."
def test_model_list_plans_every_context_with_the_install_policy(data_dir, tmp_path):
"""A pick recorded below the floor is listed (and loaded) at the floor."""
install_dir = tmp_path / "ods"
(install_dir / "data" / "models").mkdir(parents=True)
(install_dir / ".env").write_text(
"LLM_MODEL=qwen3.5-27b\n"
"GGUF_FILE=Qwen3.5-27B-Q4_K_M.gguf\n"
"SYSTEM_RAM_GB=61\n"
"MODEL_RECOMMENDED_MODEL=qwen3.5-27b\n"
"MODEL_RECOMMENDED_GGUF=Qwen3.5-27B-Q4_K_M.gguf\n"
"MODEL_RECOMMENDED_CONTEXT=32768\n",
encoding="utf-8",
)
catalog = [
raw for raw in _official_model_catalog()
if raw["id"] in {"qwen3.5-27b-q4", "gemma4-26b-a4b-q4", "phi4-q4"}
]
payload = build_models_payload(
_gpu("NVIDIA GeForce RTX 5090", 32607), None, 0, install_dir, data_dir,
catalog=catalog, evidence=[], downloaded_files_override={},
)
by_id = {model["id"]: model for model in payload["models"]}
assert by_id["qwen3.5-27b-q4"]["contextLength"] == 65536
assert by_id["gemma4-26b-a4b-q4"]["contextLength"] == 65536
assert by_id["phi4-q4"]["contextLength"] == 16384
assert by_id["phi4-q4"]["appCompatibility"]["hermesTalk"]["status"] == "unsupported"
# On a 16 GB card the 27B cannot hold the floor: the list says why up front.
small = build_models_payload(
_gpu("NVIDIA GeForce RTX 4080", 16376), None, 0, install_dir, data_dir,
catalog=catalog, evidence=[], downloaded_files_override={},
)
small_27b = next(model for model in small["models"] if model["id"] == "qwen3.5-27b-q4")
assert small_27b["contextLength"] < 65536
assert small_27b["appCompatibility"]["hermesTalk"]["code"] == "context_below_hermes_minimum"
def _memory_budget_payload(tmp_path, data_dir, monkeypatch, *, ram, backend="amd", total_mb=32768):
# The recorded host RAM must win over a smaller container memory limit.
monkeypatch.setattr("performance_oracle._system_ram_gb", lambda: 8)
install = tmp_path / "memory-budget-install"
install.mkdir()
(install / ".env").write_text(
f"SYSTEM_RAM_GB={ram}\nLLM_MODEL=qwen3.5-9b\nGGUF_FILE=Qwen3.5-9B-Q4_K_M.gguf\n"
)
catalog = _official_model_catalog()
target = next(m for m in catalog if m.get("gguf_file") == "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf")
gpu = GPUInfo(name="AMD Radeon(TM) 8060S Graphics" if backend == "amd" else backend,
memory_used_mb=9276, memory_total_mb=total_mb, memory_percent=28.3,
utilization_percent=0, temperature_c=0, gpu_backend=backend,
memory_type="discrete" if backend == "nvidia" else "unified")
payload = build_models_payload(gpu, "Qwen3.5-9B-Q4_K_M", 0, install, data_dir,
catalog=catalog, evidence=[], downloaded_files_override={})
entry = next(m for m in payload["models"] if m["id"] == target["id"])
plan = planned_model_context(normalize_catalog_entry(target), gpu, ram, preferred_context=131072)
return payload, entry, plan
def test_memory_budget_strixy_return_switch_agrees_with_activation(tmp_path, data_dir, monkeypatch):
payload, entry, plan = _memory_budget_payload(tmp_path, data_dir, monkeypatch, ram=46)
assert plan["fits"] is True
assert entry["fitsVram"] is True
assert any(o["contextLength"] == plan["context_length"] and o["fitsVram"] for o in entry["contextOptions"])
validated = ModelLibraryResponse(**payload)
assert validated.gpu.modelMemoryBudgetGb == plan["capacity_gb"] == 25.3
assert validated.gpu.vramTotal == 32
def test_memory_budget_small_shared_host_still_rejects_35b(tmp_path, data_dir, monkeypatch):
payload, entry, plan = _memory_budget_payload(tmp_path, data_dir, monkeypatch, ram=24)
assert plan["fits"] is False
assert entry["fitsVram"] is False
assert not any(o["fitsVram"] for o in entry["contextOptions"])
assert payload["gpu"]["modelMemoryBudgetGb"] == plan["capacity_gb"] == 13.2
def test_memory_budget_apple_keeps_recorded_host_ram_not_container_limit(tmp_path, data_dir, monkeypatch):
payload, entry, plan = _memory_budget_payload(tmp_path, data_dir, monkeypatch, ram=64, backend="apple", total_mb=65536)
assert entry["fitsVram"] is True
assert payload["gpu"]["modelMemoryBudgetGb"] == plan["capacity_gb"] == 35.2
def test_memory_budget_discrete_gpu_remains_bounded_by_vram(tmp_path, data_dir, monkeypatch):
payload, entry, plan = _memory_budget_payload(tmp_path, data_dir, monkeypatch, ram=128, backend="nvidia", total_mb=16384)
assert entry["fitsVram"] is False
assert payload["gpu"]["modelMemoryBudgetGb"] == plan["capacity_gb"] == 16