ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
#!/usr/bin/env python3
"""Select a pre-download ODS model from config/model-library.json.
This script is intentionally offline and deterministic. It only uses the
installer's detected hardware envelope plus the versioned model catalog; it
does not download GGUF metadata and it never treats catalog tok/s estimates as
measured performance.
By default it picks a llama.cpp (GGUF) model and never looks at the catalog's
vLLM entries (``"runtime": "vllm"``). ``--llm-runtime vllm`` picks only among
those, for one GPU named by ``--gpu``, and fails rather than fall back to a
GGUF model when none fits.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "extensions/services/dashboard-api"))
from model_memory import ( # noqa: E402
context_fitting_model as context_fitting_model,
estimate_model_memory,
estimated_context_kv_gb as estimated_context_kv_gb,
estimated_param_billions as estimated_param_billions,
memory_metadata,
required_model_memory_gb,
vllm_memory_share,
)
from model_selection import ( # noqa: E402
POLICY,
Candidate,
check_fit,
family_allowed as family_allowed,
hardware_matching_profiles as hardware_matching_profiles,
install_recommendation_allowed as install_recommendation_allowed,
list_value as list_value,
matching_runtime_profile as matching_runtime_profile,
memory_class,
normalize_backend,
normalize_host_arch as normalize_host_arch,
normalize_key as normalize_key,
rank_catalog_models,
size_within_ceiling as size_within_ceiling,
value_enabled as value_enabled,
)
from model_selection import pixel_agent_ready as _pixel_agent_ready # noqa: E402
from model_selection import usable_memory_gb as _usable_memory_gb # noqa: E402
from model_vllm import ( # noqa: E402
GPU_FREE_QUERY,
GPU_QUERY,
START_MARGIN_MIB,
Candidate as VllmCandidate,
Gpu,
free_needed_mib,
gpu_fits,
gpu_hardware_fits,
is_vllm_entry,
manifest_bytes,
memory_share,
model_dir_name,
parse_gpu,
rank_entries,
requirement_text,
vllm_args_text,
)
VRAM_FIT_TOLERANCE_GB = 0.25
PIXEL_AGENT_POLICY = "pixel-agent-capability-v1"
SPARK_AARCH64_POLICY = "spark-aarch64-nv-ultra-a3b-v1"
SPARK_AARCH64_MODEL_ID = "qwen3.6-35b-a3b-ud-q4"
# Unified-memory hosts (Strix Halo SH_LARGE, future AMD/NV unified-memory
# tiers) hit the same coder-next correctness pathology as Spark aarch64.
# Until upstream fixes coder-next on unified-memory backends, route the
# qwen profile to the same 35B-A3B substitution used for Spark — same
# model id, separate policy tag so the recommendation_reason is honest
# about why the substitution fired.
UNIFIED_MEMORY_POLICY = "unified-memory-coder-next-a3b-v1"
UNIFIED_MEMORY_MODEL_ID = SPARK_AARCH64_MODEL_ID
VLLM_POLICY = "vllm-gpu-fit-v1"
VLLM_SOURCE = "catalog_vllm_pre_download"
EXIT_NO_FIT = 2
EXIT_CHECK_FIT_FAILED = 3
# --llm-runtime vllm: an entry fits the GPUs, but other programs hold the
# memory vLLM claims at start (--gpu rows with free memory).
EXIT_GPU_BUSY = 4
def normalize_profile(value: str | None) -> str:
key = normalize_key(value or "qwen")
if key in {"gemma", "gemma4", "gemma-4"}:
return "gemma4"
if key == "auto":
return "auto"
return "qwen"
def effective_profile(profile: str, backend: str, tier: str) -> str:
if profile != "auto":
return profile
if normalize_key(tier) in {"cloud", "0", "t0"}:
return "qwen"
return "gemma4" if normalize_key(backend) in {"apple", "nvidia", "sycl"} else "qwen"
def normalize_model(raw: dict[str, Any]) -> dict[str, Any] | None:
gguf_parts = raw.get("gguf_parts") if isinstance(raw.get("gguf_parts"), list) else []
gguf = raw.get("gguf") or raw.get("gguf_file")
if not gguf and gguf_parts and isinstance(gguf_parts[0], dict):
gguf = gguf_parts[0].get("file")
model_id = raw.get("id") or raw.get("llm_model_name") or raw.get("name") or gguf
if not model_id or not gguf:
return None
try:
size_mb = float(raw.get("size_mb") or 0)
except (TypeError, ValueError):
size_mb = 0.0
try:
vram_required = float(raw.get("vram_required_gb") or 0)
except (TypeError, ValueError):
vram_required = 0.0
try:
context_length = int(raw.get("context_length") or 0)
except (TypeError, ValueError):
context_length = 0
app_compatibility = (
raw.get("app_compatibility")
if isinstance(raw.get("app_compatibility"), dict)
else {}
)
agent_viability = (
app_compatibility.get("agent_viability")
if isinstance(app_compatibility.get("agent_viability"), dict)
else {}
)
pixel_agent = (
app_compatibility.get("pixel_agent")
if isinstance(app_compatibility.get("pixel_agent"), dict)
else {}
)
return {
**memory_metadata(raw),
"id": str(model_id),
"name": raw.get("name") or str(model_id),
"family": raw.get("family") or "",
"llm_model_name": raw.get("llm_model_name") or str(model_id),
"gguf_file": str(gguf),
"gguf_url": raw.get("gguf_url") or "",
"gguf_sha256": raw.get("gguf_sha256") or "",
"gguf_parts": gguf_parts,
"size_mb": size_mb,
"vram_required_gb": vram_required,
"context_length": context_length,
"quantization": raw.get("quantization") or "",
"specialty": raw.get("specialty") or "General",
"llama_server_image": raw.get("llama_server_image") or "",
"install_recommendation": value_enabled(raw.get("install_recommendation", True)),
"selection": raw.get("selection") if isinstance(raw.get("selection"), dict) else {},
"agent_viability_status": normalize_key(agent_viability.get("status")),
"pixel_agent_status": normalize_key(pixel_agent.get("status")),
"runtime_profiles": raw.get("runtime_profiles") if isinstance(raw.get("runtime_profiles"), list) else [],
}
def curated_source_allowed(model: dict[str, Any]) -> bool:
"""Return whether a catalog record is eligible for curated selection."""
return str(model.get("source") or "").strip().lower() in {"", "curated"}
def load_catalog(path: Path) -> list[dict[str, Any]]:
"""The curated llama.cpp (GGUF) models; vLLM entries are left to _vllm_main."""
with path.open("r", encoding="utf-8") as fh:
data = json.load(fh)
return [
model for model in (
normalize_model(raw)
for raw in data.get("models", [])
if curated_source_allowed(raw) and not is_vllm_entry(raw)
)
if model is not None
]
def usable_memory_gb(backend: str, memory_type: str, vram_mb: int, ram_gb: int) -> tuple[float, str]:
return _usable_memory_gb(backend, memory_type, vram_mb, ram_gb)
def fits(required_gb: float, capacity_gb: float) -> bool:
"""Legacy tolerance rule; the ranker applies model_memory.memory_fits."""
return required_gb <= capacity_gb + VRAM_FIT_TOLERANCE_GB
def selector_required_memory_gb(model: dict[str, Any]) -> float:
return required_model_memory_gb(model)
def effective_context_length(model: dict[str, Any], runtime_profile: dict[str, Any] | None = None) -> int:
if runtime_profile and runtime_profile.get("context_length"):
return int(runtime_profile["context_length"])
return int(model.get("context_length") or 0)
def effective_required_memory_gb(model: dict[str, Any],
runtime_profile: dict[str, Any] | None = None) -> float:
selection = model.get("_selection")
if isinstance(selection, dict) and selection.get("required_gb") is not None:
return float(selection["required_gb"])
return required_model_memory_gb(
model,
context_length=effective_context_length(model, runtime_profile),
runtime_profile=runtime_profile,
)
def pixel_agent_ready(model: dict[str, Any]) -> bool:
"""Require an explicit real-Pixel capability verdict for the Pixel route."""
return _pixel_agent_ready(model)
def rank_candidates(catalog: list[dict[str, Any]], capacity_gb: float, profile: str,
installable_only: bool, backend: str, memory_type: str,
vram_mb: int, ram_gb: int, host_arch: str,
max_size_mb: float = 0,
agent_ready_only: bool = False,
min_context: int = 0,
require_min_context: bool = False,
include_size_tiebreak: bool = True) -> list[Candidate]:
return rank_catalog_models(
catalog,
capacity_gb=capacity_gb,
profile=profile,
installable_only=installable_only,
backend=backend,
memory_type=memory_type,
vram_mb=vram_mb,
ram_gb=ram_gb,
host_arch=host_arch,
max_size_mb=max_size_mb,
agent_ready_only=agent_ready_only,
min_context=min_context,
require_min_context=require_min_context,
include_size_tiebreak=include_size_tiebreak,
)
def rank_models(catalog: list[dict[str, Any]], capacity_gb: float, profile: str,
installable_only: bool, backend: str, memory_type: str,
vram_mb: int, ram_gb: int, host_arch: str,
max_size_mb: float = 0,
agent_ready_only: bool = False,
min_context: int = 0,
require_min_context: bool = False) -> list[dict[str, Any]]:
"""Ranked models at their planned context (see model_selection)."""
return [
candidate.as_model()
for candidate in rank_candidates(
catalog, capacity_gb, profile, installable_only, backend,
memory_type, vram_mb, ram_gb, host_arch, max_size_mb,
agent_ready_only, min_context, require_min_context,
)
]
def arch_policy_model(catalog: list[dict[str, Any]], tier: str, profile: str,
host_arch: str, memory_type: str,
installable_only: bool,
selected_model: dict[str, Any] | None = None,
backend: str = "") -> tuple[dict[str, Any] | None, str | None]:
"""Return (model, policy_tag) for an architecture-specific override, or (None, None).
Routes that substitute coder-next with Qwen3.6-35B-A3B-UD on unified-memory
hosts (coder-next produces all-? tokens on those backends; see the
in-source notes in installers/lib/tier-map.sh NV_ULTRA + SH_LARGE blocks):
- nv-ultra + qwen + arm64: Spark / GB10 Grace Blackwell.
- sh-large + amd + unified: every Strix Halo SH_LARGE host, whatever
ranks first, matching installers/windows/lib/tier-map.ps1.
- any-tier + qwen + memory_type=unified when coder-next ranks first:
future unified-memory NV/AMD tiers. Memory-type is the authoritative
signal (not arch or tier alone) because that's the actual
characteristic that triggers the pathology.
"""
if profile != "qwen":
return None, None
is_spark_aarch64 = (
normalize_key(tier) == "nv-ultra"
and normalize_host_arch(host_arch) == "arm64"
)
is_unified = normalize_key(memory_type) == "unified"
is_strix_large = (
is_unified
and normalize_key(tier) == "sh-large"
and normalize_backend(backend) == "amd"
)
is_unified_coder_next = (
is_unified
and selected_model is not None
and is_spark_aarch64_excluded_model(selected_model)
)
if not (is_spark_aarch64 or is_strix_large or is_unified_coder_next):
return None, None
for model in catalog:
if installable_only and not install_recommendation_allowed(model):
continue
if normalize_key(model.get("id")) == normalize_key(SPARK_AARCH64_MODEL_ID):
policy = SPARK_AARCH64_POLICY if is_spark_aarch64 else UNIFIED_MEMORY_POLICY
return model, policy
return None, None
def is_spark_aarch64_excluded_model(model: dict[str, Any]) -> bool:
"""True if `model` is the coder-next entry that we route around on
unified-memory backends. Function name preserved for backwards compat
with existing callers; the broader semantic is "excluded on unified
memory" (see arch_policy_model)."""
return normalize_key(model.get("llm_model_name")) == "qwen3-coder-next"
def shell_value(value: Any) -> str:
text = str(value or "")
text = (
text.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("$", "\\$")
.replace("`", "\\`")
)
return f'"{text}"'
def _requirement_breakdown(model: dict[str, Any]) -> str:
selection = model.get("_selection") if isinstance(model.get("_selection"), dict) else {}
source = selection.get("estimate_source")
estimate = selection.get("estimate") if isinstance(selection.get("estimate"), dict) else {}
if source == "architecture" and estimate:
parts = [
f"weights {estimate.get('weights_gib', 0):.2f}",
f"KV {estimate.get('kv_gib', 0):.2f}",
]
if estimate.get("recurrent_state_gib"):
parts.append(f"recurrent state {estimate['recurrent_state_gib']:.2f}")
parts.append(f"overhead {estimate.get('compute_overhead_gib', 0):.2f}")
if selection.get("memory_class") == "cpu" and estimate.get("host_checkpoint_gib"):
parts.append(f"context checkpoints {estimate['host_checkpoint_gib']:.2f}")
return " + ".join(parts)
if source == "runtime-profile":
return "measured runtime-profile budget"
return "catalog estimate including context/KV"
def _margin_text(model: dict[str, Any]) -> str:
selection = model.get("_selection") if isinstance(model.get("_selection"), dict) else {}
margin = float(selection.get("fit_margin_gb") or 0)
if margin < 0:
return f"within the {abs(margin):g} GiB catalog tolerance"
if margin > 0:
return f"leaving at least {margin:g} GiB free"
return "within budget"
def recommendation_reason(model: dict[str, Any], capacity_gb: float, memory_label: str,
backend: str, confidence: str) -> str:
runtime_profile = model.get("_runtime_profile") if isinstance(model.get("_runtime_profile"), dict) else None
context_k = int(effective_context_length(model, runtime_profile) / 1024)
required = effective_required_memory_gb(model, runtime_profile)
selection = model.get("_selection") if isinstance(model.get("_selection"), dict) else {}
mclass = selection.get("memory_class") or memory_class(backend, "", 0)
if runtime_profile:
label = runtime_profile.get("label") or runtime_profile.get("id") or "advanced runtime profile"
runtime = runtime_profile.get("runtime") or "llama.cpp"
ram_note = (
f" plus {runtime_profile['system_ram_min_gb']}GB system RAM"
if runtime_profile.get("system_ram_min_gb") is not None
else ""
)
return (
f"Curated runtime fit ({POLICY}): {model['name']} is the highest-priority "
f"installable model for {mclass} memory; it uses {label} via {runtime}, "
f"needs about {required:g} GiB ({_requirement_breakdown(model)}){ram_note}, "
f"fits {capacity_gb:.1f} GiB {memory_label} on {backend} "
f"({_margin_text(model)}), and gives {context_k}K context. "
f"Throughput still requires a local benchmark after first launch."
)
return (
f"Curated fit ({POLICY}): {model['name']} is the highest-priority installable "
f"model for {mclass} memory that fits {capacity_gb:.1f} GiB {memory_label} on "
f"{backend} ({_margin_text(model)}) at {context_k}K context; it needs about "
f"{required:g} GiB ({_requirement_breakdown(model)}). "
f"Throughput requires a local benchmark after first launch."
)
def arch_policy_reason(model: dict[str, Any], capacity_gb: float,
memory_label: str, policy_tag: str) -> str:
runtime_profile = model.get("_runtime_profile") if isinstance(model.get("_runtime_profile"), dict) else None
context_k = int(effective_context_length(model, runtime_profile) / 1024)
required = effective_required_memory_gb(model, runtime_profile)
if policy_tag == UNIFIED_MEMORY_POLICY:
rationale = (
"is selected for unified-memory hosts (e.g. Strix Halo, future "
"AMD/NV unified-memory tiers) because qwen3-coder-next produces "
"all-`?` tokens on unified-memory backends"
)
else:
rationale = (
"is selected for arm64 NV_ULTRA Spark-class NVIDIA hosts because "
"qwen3-coder-next is excluded on this architecture by the tier map"
)
return (
f"Arch-aware catalog policy ({policy_tag}): {model['name']} "
f"{rationale}. It needs about {required:g} GiB "
f"({_requirement_breakdown(model)}), fits {capacity_gb:.1f} GiB "
f"{memory_label}, and gives {context_k}K context. Throughput requires "
f"a local benchmark after first launch."
)
def _alternative_payload(model: dict[str, Any]) -> dict[str, Any]:
runtime_profile = model.get("_runtime_profile") if isinstance(model.get("_runtime_profile"), dict) else None
selection = model.get("_selection") if isinstance(model.get("_selection"), dict) else {}
return {
"id": model["id"],
"name": model["name"],
"gguf": model["gguf_file"],
"vram_required_gb": model["vram_required_gb"],
"estimated_required_gb": effective_required_memory_gb(model, runtime_profile),
"context_length": effective_context_length(model, runtime_profile),
"specialty": model["specialty"],
"runtime_profile": (runtime_profile or {}).get("id"),
"meets_min_context": selection.get("meets_min_context"),
"estimate_source": selection.get("estimate_source"),
"estimate": selection.get("estimate"),
}
def _check_fit_main(args: argparse.Namespace, catalog: list[dict[str, Any]],
capacity_gb: float) -> int:
model_key = normalize_key(args.model_id)
model = next(
(
item for item in catalog
if model_key in {
normalize_key(item.get("id")),
normalize_key(item.get("llm_model_name")),
normalize_key(item.get("gguf_file")),
}
),
None,
)
if model is None:
print(f"error: model {args.model_id!r} is not in the catalog", file=sys.stderr)
return 1
runtime_profile = None
if args.runtime_profile:
runtime_profile = next(
(
profile for profile in model.get("runtime_profiles") or []
if isinstance(profile, dict) and profile.get("id") == args.runtime_profile
),
None,
)
context = int(args.context or model.get("context_length") or 0)
result = check_fit(
model,
context_length=context,
capacity_gb=capacity_gb,
mclass=memory_class(args.backend, args.memory_type, args.vram_mb),
runtime_profile=runtime_profile,
)
print(json.dumps(result, indent=2))
return 0 if result["fits"] else EXIT_CHECK_FIT_FAILED
def _vllm_reason(candidate: VllmCandidate, min_context: int) -> str:
model, gpu = candidate.model, candidate.gpu
weights_gib = estimate_model_memory(model).weights_gib
share = vllm_memory_share(model)
room_gib = gpu.memory_mib * share / 1024.0 - weights_gib
reason = (
f"Curated vLLM fit ({VLLM_POLICY}): {model['name']} is the highest-priority vLLM "
f"catalog model for GPU {gpu.uuid} (compute capability {gpu.compute_capability}, "
f"{gpu.memory_mib} MiB), which meets its {model['min_gpu_mib']} MiB minimum. Its "
f"{weights_gib:.1f} GiB of weights leave about {room_gib:.1f} GiB of vLLM's "
f"{share:.2f} share of that GPU for KV cache and activations at "
f"{model['context_length'] // 1024}K context. Throughput requires a local benchmark "
f"after first launch."
)
if min_context and not candidate.meets_min_context:
reason += (
f" No vLLM catalog model serves the {min_context} context floor here; "
f"this one serves {model['context_length']}."
)
return reason
def _vllm_no_fit(pool: list[dict[str, Any]], gpus: list[Gpu], args: argparse.Namespace) -> str:
if not pool:
if args.agent_ready_only:
return "no vLLM catalog model has an explicit verified Pixel capability verdict"
return "the catalog has no vLLM models"
if not gpus:
return f"--llm-runtime vllm needs the host's NVIDIA GPUs: pass each one as --gpu ({GPU_QUERY})"
seen = ", ".join(
f"{gpu.uuid} (compute capability {gpu.compute_capability}, {gpu.memory_mib} MiB"
f"{'' if gpu.free_mib is None else f', {gpu.free_mib} MiB free'})" for gpu in gpus
)
floor = f" at the {args.min_context} context floor" if args.require_min_context and args.min_context else ""
needs = "; ".join(f"{model['id']} needs {requirement_text(model)}" for model in pool)
return f"no vLLM catalog model fits this host{floor}. GPUs: {seen}. {needs}"
def _vllm_busy(candidate: VllmCandidate, gpus: list[Gpu]) -> str:
"""Why the best entry for the GPUs' hardware cannot start: their memory is in use."""
model = candidate.model
freest = max((gpu for gpu in gpus if gpu_hardware_fits(model, gpu)),
key=lambda gpu: gpu.free_mib or 0)
return (
f"other programs use the GPU memory the vLLM engine needs: {model['name']} needs "
f"{free_needed_mib(model, freest)} MiB free when it starts ({memory_share(model):.2f} of the GPU "
f"plus {START_MARGIN_MIB} MiB for its CUDA context), and the freest GPU it fits, "
f"{freest.uuid}, has {freest.free_mib} MiB free"
)
def _vllm_busy_note(hardware_pick: VllmCandidate, gpus: list[Gpu]) -> str:
"""Names the entry the GPUs' hardware alone would get when its GPU's memory is in use."""
gpu = next(gpu for gpu in gpus if gpu.uuid == hardware_pick.gpu.uuid)
if gpu_fits(hardware_pick.model, gpu):
return ""
return (f" {hardware_pick.model['name']} would run on GPU {gpu.uuid}, but other programs use "
f"the memory it needs there.")
def _vllm_payload(ranked: list[VllmCandidate], reason: str, min_context: int, host_arch: str) -> dict[str, Any]:
selected = ranked[0]
model, gpu = selected.model, selected.gpu
return {
"policy": VLLM_POLICY,
"source": VLLM_SOURCE,
"confidence": "high",
"runtime": "vllm",
"host_arch": normalize_host_arch(host_arch),
"gpu": {key: value for key, value in gpu._asdict().items() if value is not None},
"memory_capacity_gb": round(gpu.memory_mib / 1024.0, 1),
"memory_label": "GPU memory",
"memory_class": "discrete",
"min_context": min_context,
"meets_min_context": selected.meets_min_context,
"selected": {
**{key: value for key, value in model.items() if key != "files"},
"file_count": len(model["files"]),
"model_dir": model_dir_name(model),
"estimate": estimate_model_memory(model).as_dict(),
},
"reason": reason,
"alternatives": [
{"id": item.model["id"], "name": item.model["name"],
"context_length": item.model["context_length"], "min_gpu_mib": item.model["min_gpu_mib"],
"gpu": item.gpu.uuid, "model_dir": model_dir_name(item.model)}
for item in ranked[:3]
],
}
def _vllm_env(ranked: list[VllmCandidate], reason: str) -> dict[str, Any]:
"""The installer contract for a vLLM pick (VLLM_* are .env keys as named).
VLLM_GPU_MEMORY_UTILIZATION is always the pick's own share (its entry's,
else 0.9), so a newly picked model never inherits the share an earlier
model left in .env. A rerun that keeps its model keeps its .env share
instead (phase 02's keep path).
"""
model, gpu = ranked[0].model, ranked[0].gpu
return {
"LLM_MODEL": model["llm_model_name"],
# A vLLM pick has no GGUF: clear the tier map's so nothing downloads one.
"GGUF_FILE": "",
"GGUF_URL": "",
"GGUF_SHA256": "",
"MAX_CONTEXT": model["context_length"],
"LLM_MODEL_SIZE_MB": int(round(manifest_bytes(model) / 1_000_000)),
"MODEL_RECOMMENDATION_SOURCE": VLLM_SOURCE,
"MODEL_RECOMMENDATION_POLICY": VLLM_POLICY,
"MODEL_RECOMMENDATION_CONFIDENCE": "high",
"MODEL_RECOMMENDATION_REASON": reason,
"MODEL_RECOMMENDED_ALTERNATIVES": ";".join(
f"{item.model['id']}:{item.model['context_length']}:{item.model['min_gpu_mib'] / 1024.0:g}"
for item in ranked[:3]
),
"PIXEL_AGENT_MODEL_READY": "true" if pixel_agent_ready(model) else "false",
"VLLM_MODEL_DIR": model_dir_name(model),
"VLLM_SERVED_MODEL": model["served_model_name"],
"VLLM_ARGS": vllm_args_text(model),
"VLLM_GPU_UUID": gpu.uuid,
"VLLM_GPU_MEMORY_UTILIZATION": f"{vllm_memory_share(model):g}",
}
def _vllm_main(args: argparse.Namespace) -> int:
"""``--llm-runtime vllm``: pick the best vLLM entry for one GPU, never a GGUF."""
if args.check_fit:
print("error: --check-fit checks llama.cpp models; a vLLM entry serves only "
"the context its --max-model-len fixes", file=sys.stderr)
return 1
try:
gpus = [parse_gpu(text) for text in args.gpu]
except ValueError as exc:
print(f"error: --gpu {exc}", file=sys.stderr)
return EXIT_NO_FIT
if normalize_backend(args.backend) != "nvidia":
print(f"error: vLLM models need NVIDIA GPUs; the detected backend is {args.backend}", file=sys.stderr)
return EXIT_NO_FIT
with args.catalog.open("r", encoding="utf-8") as fh:
pool = [model for model in json.load(fh).get("models", []) if is_vllm_entry(model)]
if args.agent_ready_only:
pool = [model for model in pool if pixel_agent_ready(model)]
min_context = max(int(args.min_context or 0), 0)
ranked = rank_entries(pool, gpus, min_context=min_context, require_min_context=args.require_min_context,
installable_only=args.installable_only)
# The same ranking with free memory unknown: what the GPUs' hardware alone gets.
by_hardware = rank_entries(pool, [gpu._replace(free_mib=None) for gpu in gpus], min_context=min_context,
require_min_context=args.require_min_context,
installable_only=args.installable_only)
if not ranked:
if by_hardware:
print(f"error: {_vllm_busy(by_hardware[0], gpus)}", file=sys.stderr)
return EXIT_GPU_BUSY
print(f"error: {_vllm_no_fit(pool, gpus, args)}", file=sys.stderr)
return EXIT_NO_FIT
reason = _vllm_reason(ranked[0], min_context) + _vllm_busy_note(by_hardware[0], gpus)
if not args.env:
print(json.dumps(_vllm_payload(ranked, reason, min_context, args.host_arch), indent=2))
return 0
for key, value in _vllm_env(ranked, reason).items():
print(f"{key}={shell_value(value)}")
return 0
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--catalog", required=True, type=Path)
parser.add_argument("--backend", default="unknown")
parser.add_argument("--memory-type", default="discrete")
parser.add_argument("--vram-mb", type=int, default=0)
parser.add_argument("--ram-gb", type=int, default=0)
parser.add_argument("--profile", default="qwen")
parser.add_argument("--tier", default="1")
parser.add_argument(
"--max-size-mb", type=float, default=0,
help="Optional ceiling on selected model size_mb, e.g. the tier map's "
"LLM_MODEL_SIZE_MB. 0 (default) leaves selection unbounded.",
)
parser.add_argument("--host-arch", default="unknown")
parser.add_argument("--installable-only", action="store_true")
parser.add_argument(
"--agent-ready-only",
action="store_true",
help="Select only models with an explicit verified Pixel capability verdict; "
"used for the Pixel default route.",
)
parser.add_argument(
"--min-context", type=int, default=0,
help="Soft context floor (the installers pass Hermes's 65536): a model "
"that fits at or above it outranks every model that does not.",
)
parser.add_argument(
"--require-min-context", action="store_true",
help="Make --min-context a hard floor: exit 2 when nothing fits at it.",
)
parser.add_argument(
"--check-fit", action="store_true",
help="Check one model instead of ranking: prints JSON and exits 0 when "
"--model-id fits at --context, 3 when it does not.",
)
parser.add_argument("--model-id", default="")
parser.add_argument("--context", type=int, default=0)
parser.add_argument("--runtime-profile", default="")
parser.add_argument(
"--llm-runtime", choices=("llama-server", "vllm"), default="llama-server",
help="The model server (.env LLM_BACKEND). llama-server (default) picks a GGUF "
"and never a vLLM entry; vllm picks only vLLM entries, for one --gpu, and "
"exits 2 when none fits, or 4 when one fits the GPUs but other programs "
"hold the memory vLLM claims at start. With vllm the fit comes from --gpu "
"alone: --vram-mb, --ram-gb, --memory-type, --tier, --profile and "
"--max-size-mb are not read.",
)
parser.add_argument(
"--gpu", action="append", default=[], metavar="UUID,COMPUTE_CAP,MEMORY_MIB[,FREE_MIB]",
help=f"One NVIDIA GPU, a row of `{GPU_QUERY}` or of `{GPU_FREE_QUERY}`; repeat "
"per GPU. With the free memory, a GPU fits only while vLLM can start there: "
f"its share of the GPU's total memory plus {START_MARGIN_MIB} MiB free. "
"Read only with --llm-runtime vllm.",
)
parser.add_argument("--env", action="store_true", help="print shell assignments")
args = parser.parse_args(argv)
if args.llm_runtime == "vllm":
return _vllm_main(args)
catalog = load_catalog(args.catalog)
if not catalog:
print("error: model catalog is empty (no usable models)", file=sys.stderr)
return 1
profile = effective_profile(normalize_profile(args.profile), args.backend, args.tier)
capacity_gb, memory_label = usable_memory_gb(args.backend, args.memory_type, args.vram_mb, args.ram_gb)
if args.check_fit:
if not args.model_id:
print("error: --check-fit needs --model-id", file=sys.stderr)
return 1
return _check_fit_main(args, catalog, capacity_gb)
mclass = memory_class(args.backend, args.memory_type, args.vram_mb)
confidence = "high" if args.backend not in {"unknown", "none"} and capacity_gb > 0 else "medium"
min_context = max(int(args.min_context or 0), 0)
ranked = rank_models(
catalog,
capacity_gb,
profile,
args.installable_only,
args.backend,
args.memory_type,
args.vram_mb,
args.ram_gb,
args.host_arch,
args.max_size_mb,
args.agent_ready_only,
min_context,
args.require_min_context,
)
if not ranked:
if args.agent_ready_only:
message = "no explicitly verified Pixel agent model fits the detected hardware"
elif args.require_min_context and min_context:
message = f"no installable model fits the detected hardware at {min_context} context"
else:
message = "no installable model fits the detected hardware runtime profiles"
print(f"error: {message}", file=sys.stderr)
return EXIT_NO_FIT
arch_selected, arch_policy_tag = (None, None)
if not args.agent_ready_only:
arch_selected, arch_policy_tag = arch_policy_model(
catalog, args.tier, profile, args.host_arch, args.memory_type,
args.installable_only, ranked[0], backend=args.backend,
)
if arch_selected:
arch_candidates = rank_models(
[arch_selected], capacity_gb, profile, args.installable_only,
args.backend, args.memory_type, args.vram_mb, args.ram_gb, args.host_arch,
0, False, min_context, args.require_min_context,
)
arch_selected = arch_candidates[0] if arch_candidates else None
if arch_selected:
selected = arch_selected
alternatives = [selected] + [
model for model in ranked
if model["id"] != selected["id"] and not is_spark_aarch64_excluded_model(model)
][:2]
policy = f"{POLICY}+{arch_policy_tag}"
source = "catalog_arch_policy_pre_download"
reason = arch_policy_reason(selected, capacity_gb, memory_label, arch_policy_tag)
else:
selected = ranked[0]
alternatives = ranked[:3]
policy = f"{POLICY}+{PIXEL_AGENT_POLICY}" if args.agent_ready_only else POLICY
source = "catalog_runtime_profile_pre_download" if selected.get("_runtime_profile") else "catalog_fit_pre_download"
reason = recommendation_reason(selected, capacity_gb, memory_label, args.backend, confidence)
if args.agent_ready_only:
reason += " Pixel default selection requires an explicit verified Pixel capability verdict."
if args.max_size_mb > 0 and size_within_ceiling(selected, args.max_size_mb):
reason += (
f" Bounded by --tier {args.tier}'s model size ceiling "
f"({args.max_size_mb:g}MB); use ODS_DISABLE_CATALOG_MODEL_SELECTOR=true "
f"to bypass."
)
elif args.max_size_mb > 0 and args.agent_ready_only:
reason += (
f" Pixel capability readiness overrides --tier {args.tier}'s "
f"{args.max_size_mb:g}MB model size preference because no verified "
"agent model fits beneath it."
)
elif args.max_size_mb > 0:
reason += (
f" No installable model fits beneath --tier {args.tier}'s "
f"{args.max_size_mb:g}MB model size preference, so it was relaxed."
)
selected_selection = selected.get("_selection") if isinstance(selected.get("_selection"), dict) else {}
if min_context and not selected_selection.get("meets_min_context", True):
reason += (
f" No installable model fits this hardware at the {min_context} context floor; "
f"this is the largest context that fits."
)
selected_public = {key: value for key, value in selected.items() if not key.startswith("_")}
payload = {
"policy": policy,
"source": source,
"confidence": confidence,
"profile": profile,
"host_arch": normalize_host_arch(args.host_arch),
"memory_capacity_gb": round(capacity_gb, 1),
"memory_label": memory_label,
"memory_class": mclass,
"fit_margin_gb": selected_selection.get("fit_margin_gb"),
"min_context": min_context,
"meets_min_context": selected_selection.get("meets_min_context", True),
"selected": selected_public,
"reason": reason,
"alternatives": [_alternative_payload(model) for model in alternatives],
}
if not args.env:
print(json.dumps(payload, indent=2))
return 0
alt_value = ";".join(
f"{m['id']}:{effective_context_length(m, m.get('_runtime_profile'))}:{effective_required_memory_gb(m, m.get('_runtime_profile')):g}"
for m in alternatives
)
runtime_profile = selected.get("_runtime_profile") if isinstance(selected.get("_runtime_profile"), dict) else None
env = {
"LLM_MODEL": selected["llm_model_name"],
"GGUF_FILE": selected["gguf_file"],
"GGUF_URL": selected["gguf_url"],
"GGUF_SHA256": selected["gguf_sha256"],
"MAX_CONTEXT": effective_context_length(selected, runtime_profile),
"LLM_MODEL_SIZE_MB": int(round(float(selected["size_mb"]))),
"MODEL_RECOMMENDATION_SOURCE": payload["source"],
"MODEL_RECOMMENDATION_POLICY": payload["policy"],
"MODEL_RECOMMENDATION_CONFIDENCE": payload["confidence"],
"MODEL_RECOMMENDATION_REASON": payload["reason"],
"MODEL_RECOMMENDED_ALTERNATIVES": alt_value,
"PIXEL_AGENT_MODEL_READY": "true" if pixel_agent_ready(selected) else "false",
}
if runtime_profile:
env["MODEL_RUNTIME_PROFILE"] = runtime_profile.get("id", "")
env["MODEL_RUNTIME_PROFILE_LABEL"] = runtime_profile.get("label", "")
env["MODEL_RUNTIME_PROFILE_SOURCE"] = runtime_profile.get("source_url", "")
if runtime_profile.get("llama_server_image"):
env["LLAMA_SERVER_IMAGE"] = runtime_profile["llama_server_image"]
for key, value in (runtime_profile.get("env") or {}).items():
if value is not None:
env[str(key)] = value
elif selected.get("llama_server_image"):
env["LLAMA_SERVER_IMAGE"] = selected["llama_server_image"]
for key, value in env.items():
print(f"{key}={shell_value(value)}")
return 0
if __name__ == "__main__":
sys.exit(main())