ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
#!/usr/bin/env python3
"""Safely recover a retained active model contract during ODS upgrades.
The installer owns recommendations, while the Dashboard owns the model the
operator has activated. A rerun must not silently replace that active model.
The default mode accepts only an installed, catalog-pinned local GGUF. The
host-native mode (``--native-llm``) preserves a retained, catalog-pinned
projection of the model an ODS-owned llama-server on the Windows host serves,
without pretending the Linux host owns that Windows model artifact. Two more
modes record that projection: one for a fresh install (from the model the
Windows host serves), one to repair the mismatched fields earlier fresh
installs wrote. Every mode emits a small allowlisted dotenv fragment without
``eval``. The default mode also keeps the settings model activation owns for
the same model on this host's llama-server container: an import's vision
projector and a fixed chat template, each only while it still checks out.
On a vLLM install (``LLM_BACKEND=vllm``) the default mode keeps the vLLM
route, its catalog snapshot and GPU instead (``preserved_vllm_contract``),
and exits 2 rather than let the rerun fall back to a llama.cpp model.
Round F serves every model as ``--alias <GGUF_FILE>``, so a served model id is
the GGUF filename. For one release the retired Lemonade forms of the same id
(``<stem>``, ``extra.<GGUF>``, ``user.<GGUF>``), the pre-round-F ``.env``
shapes (``ODS_MODE=lemonade``) and the pre-round-F flag names
(``--external-lemonade``, ``--project-external-lemonade``,
``--repair-external-lemonade``) are still accepted.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import platform
import re
import shlex
import stat
import sys
from pathlib import Path
from typing import Any
RUNTIME_KEYS = (
"LLAMA_PARALLEL",
"LLAMA_SERVER_MEMORY_LIMIT",
"LLAMA_ARG_FLASH_ATTN",
"LLAMA_ARG_CACHE_TYPE_K",
"LLAMA_ARG_CACHE_TYPE_V",
"LLAMA_ARG_N_CPU_MOE",
"LLAMA_ARG_NO_CACHE_PROMPT",
"LLAMA_ARG_CHECKPOINT_EVERY_NT",
"LLAMA_ARG_CTX_CHECKPOINTS",
"LLAMA_ARG_CACHE_RAM",
"LLAMA_ARG_SPEC_TYPE",
"LLAMA_ARG_SPEC_DRAFT_N_MAX",
"LLAMA_ARG_SPEC_DRAFT_TYPE_K",
"LLAMA_ARG_SPEC_DRAFT_TYPE_V",
"LLAMA_ARG_SPLIT_MODE",
"LLAMA_ARG_TENSOR_SPLIT",
)
# A completed switchboard proof records the exact model and context, but the v1
# state contract does not record the runtime profile. After an interrupted
# installer has already replaced .env, recover only portable, catalog-owned
# memory controls from one unambiguous profile for that proven context. GPU
# placement and speculative/MoE tuning must never be inferred this way.
PORTABLE_STATE_RECOVERY_KEYS = {
"LLAMA_PARALLEL",
"LLAMA_SERVER_MEMORY_LIMIT",
"LLAMA_ARG_FLASH_ATTN",
"LLAMA_ARG_CACHE_TYPE_K",
"LLAMA_ARG_CACHE_TYPE_V",
"LLAMA_ARG_CTX_CHECKPOINTS",
"LLAMA_ARG_CACHE_RAM",
}
# Model activation (bin/ods-host-agent.py) sets LLAMA_ARG_MMPROJ to an import's
# own projector as /models/<file> and removes it for every other model.
_CONTAINER_PROJECTOR = re.compile(r"/models/([A-Za-z0-9][A-Za-z0-9._-]{0,250}\.gguf)")
def parse_dotenv(path: Path) -> dict[str, str]:
values: dict[str, str] = {}
try:
lines = path.read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeError):
return values
for raw_line in lines:
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, raw_value = line.split("=", 1)
key = key.strip()
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
continue
# Match the literal inline-comment rules used by lib/safe-env.sh.
# Enabling shlex comments globally would also cut URL fragments and
# other unquoted hashes that are part of the model contract.
raw_value = raw_value.strip()
if raw_value.startswith(("'", '"')):
comment = re.match(r"""^("(?:\\.|[^"\\])*"|'[^']*')\s+#""", raw_value)
if comment:
raw_value = comment.group(1)
else:
raw_value = raw_value.split(" #", 1)[0].rstrip()
try:
parsed = shlex.split(raw_value, comments=False, posix=True)
except ValueError:
continue
if len(parsed) <= 1:
values[key] = parsed[0] if parsed else ""
return values
def shell_value(value: Any) -> str:
text = str(value if value is not None else "")
text = (
text.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("$", "\\$")
.replace("`", "\\`")
)
return f'"{text}"'
def normalize_key(value: Any) -> str:
return re.sub(r"[^a-z0-9]+", "-", str(value or "").lower()).strip("-")
def normalize_arch(value: Any) -> str:
key = normalize_key(value)
if key in {"x86-64", "x86-64-v2", "x86-64-v3", "x86-64-v4", "amd64", "x64"}:
return "amd64"
if key in {"aarch64", "arm64"}:
return "arm64"
return key or "unknown"
def positive_float(value: Any) -> float | None:
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if math.isfinite(number) and number > 0 else None
def positive_int(value: Any) -> int | None:
try:
number = int(str(value))
except (TypeError, ValueError):
return None
return number if number > 0 else None
_HF_RESOLVE_URL = re.compile(
r"https://huggingface\.co/([A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*)"
r"/resolve/[^/?#]+/([^?#]+)"
)
def same_pinned_artifact(env_url: str, catalog_url: str, env_digest: str, catalog_digest: str) -> bool:
"""True when two Hub URLs name the same repo file and the digests agree.
Only the revision segment may differ. A different repo, path, host or a
missing/mismatched sha256 is still a different artifact.
"""
env_match = _HF_RESOLVE_URL.fullmatch(env_url.strip())
catalog_match = _HF_RESOLVE_URL.fullmatch(catalog_url.strip())
digest = env_digest.strip().lower()
return bool(
env_match
and catalog_match
and env_match.group(1).lower() == catalog_match.group(1).lower()
and env_match.group(2) == catalog_match.group(2)
and re.fullmatch(r"[0-9a-f]{64}", digest)
and digest == catalog_digest.strip().lower()
)
def load_records(catalog_path: Path, imports_path: Path | None) -> list[dict[str, Any]]:
try:
payload = json.loads(catalog_path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError):
return []
curated = payload.get("models") if isinstance(payload, dict) else None
if not isinstance(curated, list) or not all(isinstance(item, dict) for item in curated):
return []
records = list(curated)
seen_ids = {str(item.get("id") or "") for item in records}
seen_files = {str(item.get("gguf_file") or "").lower() for item in records}
if imports_path is None or not imports_path.is_file():
return records
# An unreadable or malformed import registry hides only the imports, as in
# the dashboard: the catalog (and an active catalog model) still counts.
try:
imported_payload = json.loads(imports_path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError):
return records
imported = imported_payload.get("models") if isinstance(imported_payload, dict) else None
if not isinstance(imported, list):
return records
for item in imported:
if not isinstance(item, dict):
continue
model_id = str(item.get("id") or "")
filename = str(item.get("gguf_file") or "").lower()
# Skip an import the catalog now ships (or one that is malformed), as
# the dashboard's _load_library does. Dropping every record instead
# made a rerun replace the owner's active model with the tier pick.
if (
item.get("source") != "huggingface"
or not model_id
or not filename
or model_id in seen_ids
or filename in seen_files
):
continue
records.append(item)
seen_ids.add(model_id)
seen_files.add(filename)
return records
def load_verified_active_state(path: Path | None) -> dict[str, Any] | None:
"""Return only a completed, owner-custodied local switchboard proof."""
if path is None:
return None
try:
info = path.lstat()
if (
not stat.S_ISREG(info.st_mode)
or stat.S_ISLNK(info.st_mode)
or info.st_nlink != 1
or (hasattr(os, "getuid") and info.st_uid != os.getuid())
or info.st_mode & 0o022
or info.st_size <= 0
or info.st_size > 2 * 1024 * 1024
):
return None
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError):
return None
if not isinstance(payload, dict) or payload.get("schema") != "ods.model-state.v1":
return None
active = payload.get("active")
desired = payload.get("desired")
availability = payload.get("availability")
if not isinstance(active, dict) or not isinstance(desired, dict) or not isinstance(availability, dict):
return None
proof = active.get("proof")
backend = active.get("backend")
catalog_id = active.get("catalogId")
runtime_model_id = active.get("runtimeModelId")
context_length = positive_int(active.get("contextLength"))
route_seq = positive_int(active.get("routeSeq"))
if (
payload.get("operation") is not None
or availability.get("mode") != "serve_active"
or desired.get("catalogId") != catalog_id
or route_seq is None
or positive_int(payload.get("routeSeq")) != route_seq
or not isinstance(catalog_id, str)
or not catalog_id
or not isinstance(runtime_model_id, str)
or Path(runtime_model_id).name != runtime_model_id
or not runtime_model_id.lower().endswith(".gguf")
or context_length is None
or context_length < 1024
or context_length > 9_007_199_254_740_991
or not isinstance(active.get("verifiedAt"), str)
or not active.get("verifiedAt")
or not isinstance(proof, dict)
or proof.get("completion") is not True
or proof.get("identity") != runtime_model_id
or not isinstance(backend, dict)
or backend.get("kind") != "llama-server"
or backend.get("endpointId") != "llama-server-default"
or backend.get("nativeRoute") is not None
):
return None
return {
"catalogId": catalog_id,
"runtimeModelId": runtime_model_id,
"contextLength": context_length,
}
def manifest_for(record: dict[str, Any]) -> list[dict[str, Any]] | None:
primary = str(record.get("gguf_file") or "").strip()
if not primary or Path(primary).name != primary or not primary.lower().endswith(".gguf"):
return None
raw_parts = record.get("gguf_parts")
if isinstance(raw_parts, list) and raw_parts:
artifacts = raw_parts
else:
artifacts = [
{
"file": primary,
"url": record.get("gguf_url"),
"sha256": record.get("gguf_sha256"),
"size_bytes": record.get("size_bytes"),
}
]
normalized: list[dict[str, Any]] = []
seen: set[str] = set()
for raw in artifacts:
if not isinstance(raw, dict):
return None
filename = str(raw.get("file") or "").strip()
url = str(raw.get("url") or "").strip()
digest = str(raw.get("sha256") or "").strip().lower()
if (
not filename
or Path(filename).name != filename
or filename in seen
or not filename.lower().endswith(".gguf")
or not url.startswith("https://huggingface.co/")
or (digest and not re.fullmatch(r"[0-9a-f]{64}", digest))
):
return None
expected_size = positive_int(raw.get("size_bytes"))
normalized.append(
{"file": filename, "url": url, "sha256": digest, "size_bytes": expected_size}
)
seen.add(filename)
if primary not in seen:
return None
return normalized
def profile_is_eligible(profile_record: dict[str, Any], args: argparse.Namespace) -> bool:
backend = normalize_key(args.backend)
profile_backend = normalize_key(profile_record.get("backend"))
if profile_backend and profile_backend != backend:
return False
allowed_arches = {
normalize_arch(item)
for item in (
profile_record.get("host_arch")
if isinstance(profile_record.get("host_arch"), list)
else [profile_record.get("host_arch")]
)
if item
}
if allowed_arches and normalize_arch(args.host_arch or platform.machine()) not in allowed_arches:
return False
required_memory_type = normalize_key(profile_record.get("memory_type"))
if required_memory_type and required_memory_type != normalize_key(args.memory_type):
return False
try:
if profile_record.get("vram_min_gb") is not None and args.vram_mb / 1024 < float(profile_record["vram_min_gb"]):
return False
if profile_record.get("vram_max_gb") is not None and args.vram_mb / 1024 > float(profile_record["vram_max_gb"]):
return False
if profile_record.get("system_ram_min_gb") is not None and args.ram_gb < float(profile_record["system_ram_min_gb"]):
return False
except (TypeError, ValueError):
return False
return True
def valid_runtime_value(key: str, value: str) -> bool:
if key == "LLAMA_PARALLEL":
number = positive_int(value)
return number is not None and number <= 128
if key == "LLAMA_SERVER_MEMORY_LIMIT":
return bool(re.fullmatch(r"[1-9][0-9]*(?:\.[0-9]+)?[KMGTP]?[bB]?", value))
if key == "LLAMA_ARG_FLASH_ATTN":
return value.lower() in {"auto", "on", "off", "true", "false", "0", "1"}
if key in {"LLAMA_ARG_CACHE_TYPE_K", "LLAMA_ARG_CACHE_TYPE_V", "LLAMA_ARG_SPEC_DRAFT_TYPE_K", "LLAMA_ARG_SPEC_DRAFT_TYPE_V"}:
return bool(re.fullmatch(r"[A-Za-z0-9_.-]{1,32}", value))
if key == "LLAMA_ARG_N_CPU_MOE":
return value.isdigit() and int(value) <= 4096
if key == "LLAMA_ARG_NO_CACHE_PROMPT":
return value.lower() in {"", "on", "off", "true", "false", "0", "1"}
if key == "LLAMA_ARG_CHECKPOINT_EVERY_NT":
return bool(re.fullmatch(r"-?[0-9]{1,10}", value))
if key == "LLAMA_ARG_CTX_CHECKPOINTS":
return value.isdigit() and int(value) <= 64
if key == "LLAMA_ARG_CACHE_RAM":
return value == "-1" or (value.isdigit() and int(value) <= 1048576)
if key == "LLAMA_ARG_SPEC_TYPE":
return bool(re.fullmatch(r"[A-Za-z0-9_,.-]{1,64}", value))
if key == "LLAMA_ARG_SPEC_DRAFT_N_MAX":
number = positive_int(value)
return number is not None and number <= 4096
if key == "LLAMA_ARG_SPLIT_MODE":
return value in {"none", "layer", "row"}
if key == "LLAMA_ARG_TENSOR_SPLIT":
return value == "" or bool(re.fullmatch(r"[0-9]+(?:\.[0-9]+)?(?:,[0-9]+(?:\.[0-9]+)?)*", value))
return False
def is_retained_external_lemonade(env: dict[str, str]) -> bool:
"""A pre-round-F ``.env`` whose model a Windows-hosted Lemonade served.
Readable for one release, until the installer's ``.env`` migration.
"""
return (
env.get("ODS_MODE", "").lower() == "lemonade"
and env.get("LLM_BACKEND", "").lower() == "lemonade"
and env.get("LEMONADE_EXTERNAL", "").lower() == "true"
)
def is_retained_host_native(env: dict[str, str]) -> bool:
"""A ``.env`` whose model the ODS-owned llama-server.exe on Windows serves.
The WSL Portal family: its stack reaches the Windows task through the
owned model-router, or directly at ``NATIVE_LLM_BASE_URL``, the origin
every host-native install records. Retired ``LEMONADE_*`` lines never
decide this.
"""
transport = env.get("ODS_HOST_LLM_TRANSPORT", env.get("LEMONADE_HOST_TRANSPORT", ""))
return (
env.get("ODS_MODE", "local").lower() in {"local", "lemonade"}
and env.get("LLM_BACKEND", "").lower() == "llama-server"
and (
env.get("AMD_INFERENCE_RUNTIME_MODE", "").lower() == "windows-portal-llama-server"
or transport.lower() == "model-router"
or bool(env.get("NATIVE_LLM_BASE_URL", "").strip())
)
)
def is_legacy_managed_lemonade(env: dict[str, str]) -> bool:
"""A managed Linux AMD ``.env`` written before round F (one release).
Its GGUF_FILE is in this host's model directory; after the ``.env``
migration upstream llama-server serves exactly that file.
"""
return (
env.get("ODS_MODE", "").lower() == "lemonade"
and env.get("LLM_BACKEND", "lemonade").lower() in {"lemonade", "llama-server"}
and env.get("LEMONADE_EXTERNAL", "false").lower() != "true"
and not (
env.get("AMD_INFERENCE_RUNTIME", "").lower() == "lemonade"
and env.get("AMD_INFERENCE_MANAGED", "").lower() == "false"
)
)
def lemonade_model_ids(gguf_file: str) -> set[str]:
"""The retired Lemonade ids that name a catalog GGUF (one release).
Its stem, or its ``extra.``/``user.`` alias: the matchers contract
section 6.7 keeps readable.
"""
return {Path(gguf_file).stem, f"extra.{gguf_file}", f"user.{gguf_file}"}
def served_model_ids(gguf_file: str) -> set[str]:
"""Every id that names a catalog GGUF: the ``--alias`` filename itself first."""
return {gguf_file} | lemonade_model_ids(gguf_file)
def served_model_projection(
records: list[dict[str, Any]], model_id: str, context: int | None
) -> dict[str, str] | None:
"""Describe, for .env, the catalog model a served model id names.
The Windows installer chooses and loads the model its llama-server
serves; this host only records it. The id must name exactly one catalog
GGUF, by the rule the rerun check applies, so the record always passes
that check. The context is the one the server loaded when given, else
the catalog's, and never above the model's native maximum.
"""
matches = [
item
for item in records
if isinstance(item.get("gguf_file"), str)
and Path(item["gguf_file"]).name == item["gguf_file"]
and model_id in served_model_ids(item["gguf_file"])
]
if len(matches) != 1:
return None
model = matches[0]
manifest = manifest_for(model)
llm_model = str(model.get("llm_model_name") or model.get("id") or "").strip()
if manifest is None or not llm_model:
return None
primary = next(item for item in manifest if item["file"] == model["gguf_file"])
if context is None:
context = positive_int(model.get("context_length"))
if context is None or context < 1024 or context > 9_007_199_254_740_991:
return None
native_max = positive_int(model.get("max_context_length"))
if native_max and context > native_max:
context = native_max
image = str(model.get("llama_server_image") or "")
if image and not re.fullmatch(r"[A-Za-z0-9._/@:+-]{1,300}", image):
return None
return {
"LLM_MODEL": llm_model,
"GGUF_FILE": model["gguf_file"],
"GGUF_URL": str(primary["url"]),
"GGUF_SHA256": str(primary["sha256"]),
"MAX_CONTEXT": str(context),
# The rerun check records at least 1 MB when the file is not on this host.
"LLM_MODEL_SIZE_MB": str(max(positive_int(model.get("size_mb")) or 0, 1)),
"MODEL_RUNTIME_PROFILE": "",
"MODEL_RUNTIME_PROFILE_LABEL": "",
"MODEL_RUNTIME_PROFILE_SOURCE": "",
"MODEL_SELECTION_SOURCE": "installer",
"LLAMA_SERVER_IMAGE": image,
}
def repaired_served_model_contract(
args: argparse.Namespace, served_id: str | None,
) -> tuple[dict[str, str], str, dict[str, str]] | None:
"""Re-record the mismatched model fields an earlier fresh install wrote.
Fresh Windows-hosted installs recorded this host's own catalog pick, a
model nobody serves, next to the model the Windows host serves; the
rerun check rightly refuses that. Repair only that exact shape: the
saved pick is still the installer's own recommendation (no Dashboard or
operator choice is overwritten) and the served id names exactly one
catalog model. The served model does not change, only its description.
``served_id`` is the model the Windows host serves now (its GGUF). The
pre-round-F flag passes None and the id is the ``.env``'s own retired
``LEMONADE_MODEL``; a pre-round-F ``.env`` that records another served
model than the one named is refused.
"""
env = parse_dotenv(args.env)
legacy = is_retained_external_lemonade(env)
recorded_id = env.get("LEMONADE_MODEL", "") if legacy else ""
if served_id is None:
served_id = recorded_id
active_file = env.get("GGUF_FILE", "").strip()
if not (
(legacy or is_retained_host_native(env))
and not env.get("EXTERNAL_LLM_URL")
and env.get("MODEL_SELECTION_SOURCE") == "installer"
and env.get("ODS_ACTIVE_MODEL_STORE", "default") == "default"
and served_id
and active_file
and Path(active_file).name == active_file
and served_id not in served_model_ids(active_file)
and env.get("MODEL_RECOMMENDED_GGUF") == active_file
and env.get("MODEL_RECOMMENDED_MODEL", env.get("LLM_MODEL")) == env.get("LLM_MODEL")
):
return None
contract = served_model_projection(load_records(args.catalog, args.imports), served_id, args.context)
if contract is None:
return None
if recorded_id and recorded_id not in served_model_ids(contract["GGUF_FILE"]):
return None
return env, served_id, contract
def is_vllm_runtime(env: dict[str, str]) -> bool:
"""A ``.env`` whose model the vLLM engine serves (LLM_BACKEND=vllm)."""
return env.get("LLM_BACKEND", "").strip().lower() == "vllm"
_VLLM_GPU_UUID = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*")
_VLLM_MEMORY_SHARE = re.compile(r"(?:0(?:\.[0-9]+)?|1(?:\.0+)?)")
def vllm_snapshot_installed(models_dir: Path, folder: str, files: list[dict[str, Any]]) -> bool:
"""Every manifest file is in ``models_dir/folder`` under its final name, at its size.
scripts/download-hf-manifest.py renames a file into place only after its
digest matched, so a complete set of final names at the catalog sizes is a
complete verified download. Links are refused, as the GGUF check does.
"""
try:
root = models_dir.resolve()
snapshot = models_dir / folder
if snapshot.is_symlink() or not snapshot.is_dir() or snapshot.resolve().parent != root:
return False
for item in files:
info = (snapshot / item["path"]).lstat()
if not stat.S_ISREG(info.st_mode) or info.st_size != item["size"]:
return False
if not (snapshot / item["path"]).resolve().is_relative_to(snapshot.resolve()):
return False
except (OSError, RuntimeError):
return False
return True
def preserved_vllm_contract(args: argparse.Namespace) -> dict[str, str] | None:
"""The retained vLLM route, while its catalog snapshot is still installed.
The route stays on vLLM (LLM_BACKEND=vllm) with the same model and GPU, and
clears the GGUF keys so no llama.cpp model is downloaded or started. The
model folder and served id must be the catalog entry's (a new revision is
a new download, which this never starts). VLLM_ARGS comes from this
release's catalog entry for that snapshot: its flags are measured with the
release's vLLM image, which the installer pins separately (VLLM_IMAGE).
The GPU and its memory share are the host's and stay as installed.
"""
env = parse_dotenv(args.env)
if (
not is_vllm_runtime(env)
or env.get("ODS_MODE", "local").strip().lower() != "local"
or env.get("EXTERNAL_LLM_URL")
or env.get("NATIVE_LLM_BASE_URL")
):
return None
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "extensions/services/dashboard-api"))
import model_vllm
llm_model = env.get("LLM_MODEL", "").strip()
matches = [
item for item in load_records(args.catalog, None)
if model_vllm.is_vllm_entry(item) and item.get("llm_model_name") == llm_model
]
if len(matches) != 1 or model_vllm.entry_problems(matches[0]):
return None
model = matches[0]
folder = model_vllm.model_dir_name(model)
gpu_uuid = env.get("VLLM_GPU_UUID", "")
share = env.get("VLLM_GPU_MEMORY_UTILIZATION", "")
if (
env.get("VLLM_MODEL_DIR") != folder
or env.get("VLLM_SERVED_MODEL") != model["served_model_name"]
or not _VLLM_GPU_UUID.fullmatch(gpu_uuid)
or (share and (not _VLLM_MEMORY_SHARE.fullmatch(share) or float(share) <= 0))
or not vllm_snapshot_installed(args.models_dir, folder, model["files"])
):
return None
source = env.get("MODEL_SELECTION_SOURCE", "")
contract = {
"LLM_BACKEND": "vllm",
"LLM_MODEL": str(model["llm_model_name"]),
# A vLLM route has no GGUF; empty values override the tier map's.
"GGUF_FILE": "",
"GGUF_URL": "",
"GGUF_SHA256": "",
"MAX_CONTEXT": str(model["context_length"]),
"LLM_MODEL_SIZE_MB": str(round(model_vllm.manifest_bytes(model) / 1_000_000)),
"MODEL_SELECTION_SOURCE": source if source in {"installer", "dashboard", "operator"} else "preserved-local",
"VLLM_MODEL_DIR": folder,
"VLLM_SERVED_MODEL": str(model["served_model_name"]),
"VLLM_ARGS": model_vllm.vllm_args_text(model),
"VLLM_GPU_UUID": gpu_uuid,
}
if share:
contract["VLLM_GPU_MEMORY_UTILIZATION"] = share
return contract
def is_local_runtime(env: dict[str, str]) -> bool:
"""A ``.env`` whose model this host's managed llama-server serves."""
if env.get("EXTERNAL_LLM_URL", "") or env.get("LEMONADE_EXTERNAL", "false").lower() == "true":
return False
if is_legacy_managed_lemonade(env):
return True
return (
env.get("ODS_MODE", "local").lower() == "local"
and env.get("LLM_BACKEND", "llama-server").lower() == "llama-server"
)
def carried_projector(
env: dict[str, str], model: dict[str, Any], manifest: list[dict[str, Any]], models_dir: Path,
) -> str | None:
"""The saved LLAMA_ARG_MMPROJ while it is still this model's vision projector.
Activation writes ``/models/<file>`` from the record's ``mmproj_file`` when
that file is beside the weights (ods-host-agent.py ``_model_projector_file``).
Keep exactly that value while the file is a regular file (not a link) in
the model directory, at the size the import recorded. Otherwise drop it,
as every rerun did before: the model runs without image input until its
next switch.
"""
value = env.get("LLAMA_ARG_MMPROJ", "")
if not value:
return None
match = _CONTAINER_PROJECTOR.fullmatch(value)
filename = match.group(1) if match else ""
expected_size = positive_int(model.get("mmproj_size_bytes"))
info: os.stat_result | None = None
if filename and filename == model.get("mmproj_file") and filename not in {item["file"] for item in manifest}:
try:
info = (models_dir / filename).lstat()
except OSError:
info = None
if (
info is None
or not stat.S_ISREG(info.st_mode)
or info.st_size <= 0
or (expected_size is not None and info.st_size != expected_size)
):
print(f"Not keeping LLAMA_ARG_MMPROJ={value!r}: it does not name this model's complete vision "
"projector file in its model folder. The model runs without image input until it is "
"switched to again on the Models page.", file=sys.stderr)
return None
return value
def chat_template_ships(root: Path, override: str, container_file: str) -> bool:
"""True when ``root``'s index has entry ``override`` at ``container_file`` and its file verifies."""
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "bin"))
try:
from model_profile import templates
except ImportError:
return False
if not re.fullmatch(templates.ID_PATTERN, override):
return False
try:
entry = templates.entry_by_id(templates.load_index(root), override)
if entry is None or container_file != f"/chat-templates/{entry['file']}":
return False
templates.template_path(root, entry)
except (OSError, UnicodeError, templates.TemplateIndexError):
return False
return True
def carried_chat_template(env: dict[str, str], chat_templates: Path | None) -> dict[str, str]:
"""The saved fixed chat template, both of its keys, while this release still ships it.
For the llama-server container, activation sets MODEL_CHAT_TEMPLATE_OVERRIDE
to an index entry's id and LLAMA_ARG_CHAT_TEMPLATE_FILE to
``/chat-templates/<entry file>`` together, and removes both for every
other model (ods-host-agent.py ``_chat_template_env_changes``). Keep the
pair only when ``chat_templates`` (this release's config/chat-templates,
which the container mounts at /chat-templates) still has that entry at
that file with its recorded SHA-256. Otherwise drop both, as before.
"""
override = env.get("MODEL_CHAT_TEMPLATE_OVERRIDE", "")
container_file = env.get("LLAMA_ARG_CHAT_TEMPLATE_FILE", "")
if not override and not container_file:
return {}
if chat_templates is not None and chat_template_ships(chat_templates, override, container_file):
return {"MODEL_CHAT_TEMPLATE_OVERRIDE": override, "LLAMA_ARG_CHAT_TEMPLATE_FILE": container_file}
print(f"Not keeping the fixed chat template {override!r} ({container_file!r}): this release does not "
"ship it in that form. The model runs with its own template until it is switched to again "
"on the Models page.", file=sys.stderr)
return {}
def preserved_contract(args: argparse.Namespace) -> dict[str, str] | None:
env = parse_dotenv(args.env)
# The pre-round-F name of the host-native mode stays an alias for one release.
host_native = getattr(args, "native_llm", False) or getattr(args, "external_lemonade", False)
legacy_shape = host_native and is_retained_external_lemonade(env)
if host_native:
if not (
(legacy_shape or is_retained_host_native(env))
and env.get("MODEL_SELECTION_SOURCE") in {"dashboard", "operator", "installer", "preserved-external"}
and (env.get("LEMONADE_MODEL") or not legacy_shape)
and not env.get("EXTERNAL_LLM_URL")
):
return None
elif not is_local_runtime(env):
return None
records = load_records(args.catalog, args.imports)
# The local state proof belongs to this host's llama-server. A Windows-
# hosted model is never proven here: preserve a separately validated
# .env projection; later installer health checks still prove the route.
verified_state = None if host_native else load_verified_active_state(args.state)
state_authoritative = verified_state is not None
if state_authoritative:
active_file = str(verified_state["runtimeModelId"])
matches = [
item
for item in records
if item.get("id") == verified_state["catalogId"]
and str(item.get("gguf_file") or "") == active_file
]
else:
active_file = env.get("GGUF_FILE", "").strip()
if not active_file or Path(active_file).name != active_file:
return None
if (legacy_shape or is_legacy_managed_lemonade(env)) and env.get("LEMONADE_MODEL") and (
env["LEMONADE_MODEL"] not in lemonade_model_ids(active_file)):
# A pre-round-F .env's explicit Lemonade id must identify the saved
# catalog artifact, not an unrelated model left in the same .env.
return None
served = getattr(args, "served_model", None)
if host_native and served is not None and served not in served_model_ids(active_file):
# The Windows host serves another model than the retained record.
return None
matches = [
item for item in records if str(item.get("gguf_file") or "").lower() == active_file.lower()
]
if len(matches) != 1:
return None
model = matches[0]
manifest = manifest_for(model)
if manifest is None:
return None
actual_bytes = 0
models_dir = args.models_dir
active_store_id = env.get("ODS_ACTIVE_MODEL_STORE", "default")
if host_native and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}", active_store_id):
return None
if active_store_id != "default" and not host_native:
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "extensions/services/dashboard-api"))
try:
from model_stores import active_store
models_dir = active_store(args.env.parent / "data", active_store_id)["path"]
except (ValueError, OSError, ImportError):
return None
try:
models_root = models_dir.resolve()
except (OSError, RuntimeError):
return None
for artifact in (() if host_native else manifest):
artifact_path = models_dir / artifact["file"]
try:
resolved_artifact = artifact_path.resolve()
if not resolved_artifact.is_relative_to(models_root):
return None
if not resolved_artifact.is_file() or resolved_artifact.stat().st_size <= 0:
return None
size = resolved_artifact.stat().st_size
except (OSError, RuntimeError):
return None
expected_size = artifact.get("size_bytes")
if expected_size is not None and size != expected_size:
return None
actual_bytes += size
llm_model = str(model.get("llm_model_name") or model.get("id") or "").strip()
if not llm_model or (
not state_authoritative and env.get("LLM_MODEL") and env["LLM_MODEL"] != llm_model
):
return None
primary = next(item for item in manifest if item["file"] == str(model["gguf_file"]))
for key, expected in (
("GGUF_URL", primary["url"]),
("GGUF_SHA256", primary["sha256"]),
):
if not state_authoritative and env.get(key) and env[key] != expected:
if key == "GGUF_URL" and same_pinned_artifact(
env[key], expected, env.get("GGUF_SHA256", ""), primary["sha256"]
):
# Catalog re-pin of the same Hub file (e.g. resolve/main ->
# resolve/<commit>) with an unchanged digest is not a model
# change. The contract below carries the catalog's new URL.
continue
return None
if state_authoritative:
context = int(verified_state["contextLength"])
else:
context_values = [positive_int(env.get(key)) for key in ("MAX_CONTEXT", "CTX_SIZE") if env.get(key)]
if not context_values or any(value is None for value in context_values) or len(set(context_values)) != 1:
return None
context = int(context_values[0])
# Dashboard activation accepts advanced custom contexts above the catalog's
# declared recommendation and commits only after the live runtime proves
# the exact value. Preserve that already-qualified operator choice instead
# of turning a catalog advisory into an upgrade-time hard limit.
if context < 1024 or context > 9_007_199_254_740_991:
return None
# A host-native llama-server reports the context it actually loaded the
# retained model with; that is what is served, so it replaces the saved
# value. The owner stays, and the cap below still applies.
loaded_context = getattr(args, "context", None)
if host_native and loaded_context is not None:
context = int(loaded_context)
# The owner's choice is honored up to the model's declared native
# maximum only. Above it llama.cpp caps the slot at the training context,
# so the recorded value is never served and the activation's context
# proof cannot pass (tower2 2026-09-25: qwen3-30b-a3b-q4 recorded at
# 131072 on a 40960-token GGUF). Carry the context that is served.
native_max = positive_int(model.get("max_context_length"))
if native_max and context > native_max:
context = native_max
reuse_env_runtime = (
not state_authoritative
or (
env.get("GGUF_FILE", "").strip() == active_file
and env.get("LLM_MODEL", "").strip() == llm_model
)
)
profile_id = env.get("MODEL_RUNTIME_PROFILE", "") if reuse_env_runtime else ""
runtime_profile: dict[str, Any] | None = None
runtime_defaults_profile: dict[str, Any] | None = None
if profile_id:
profiles = model.get("runtime_profiles")
if not isinstance(profiles, list):
return None
profile_matches = [
item for item in profiles if isinstance(item, dict) and str(item.get("id") or "") == profile_id
]
if len(profile_matches) != 1:
return None
if host_native or profile_is_eligible(profile_matches[0], args):
runtime_profile = profile_matches[0]
runtime_defaults_profile = runtime_profile
elif state_authoritative:
# Exact model/context proof plus a still-matching .env is stronger
# than a later best-effort hardware probe (notably under WSL). Keep
# the proven runtime values but stop claiming the old profile is a
# currently hardware-qualified selection.
runtime_defaults_profile = profile_matches[0]
profile_id = ""
else:
return None
elif state_authoritative:
profiles = model.get("runtime_profiles")
exact_context_profiles = [
item
for item in profiles if isinstance(item, dict)
and positive_int(item.get("context_length")) == context
and isinstance(item.get("id"), str)
and item.get("id")
] if isinstance(profiles, list) else []
if len(exact_context_profiles) == 1:
candidate = exact_context_profiles[0]
candidate_env = candidate.get("env")
if (
isinstance(candidate_env, dict)
and set(candidate_env) <= PORTABLE_STATE_RECOVERY_KEYS
and all(
value is not None and valid_runtime_value(key, str(value))
for key, value in candidate_env.items()
)
):
runtime_defaults_profile = candidate
candidate_matches_env = all(
env.get(key, "") == str(value)
for key, value in candidate_env.items()
) and all(
not env.get(key, "")
for key in RUNTIME_KEYS if key not in candidate_env
)
if not candidate_matches_env:
reuse_env_runtime = False
if profile_is_eligible(candidate, args):
runtime_profile = candidate
profile_id = str(candidate["id"])
declared_mb = positive_int(model.get("size_mb")) or 0
actual_mb = max(1, math.ceil(actual_bytes / (1024 * 1024)))
profile_env = (
runtime_defaults_profile.get("env")
if runtime_defaults_profile and isinstance(runtime_defaults_profile.get("env"), dict)
else {}
)
runtime_values: dict[str, str] = {}
for key in RUNTIME_KEYS:
if reuse_env_runtime and key in env:
value = env[key]
elif key in profile_env and profile_env[key] is not None:
value = str(profile_env[key])
else:
value = ""
if value and not valid_runtime_value(key, value):
return None
runtime_values[key] = value
image = ""
if runtime_profile:
image = str(runtime_profile.get("llama_server_image") or "")
image = image or str(model.get("llama_server_image") or "")
# A saved image is an explicit runtime pin, even when it currently equals
# the catalog default. Carry it only with the same retained local model
# and GPU backend; a backend/native-route change must choose its own image.
saved_backend = normalize_key(env.get("GPU_BACKEND"))
if (
reuse_env_runtime and not host_native and not env.get("NATIVE_LLM_BASE_URL")
and env.get("LLM_BACKEND") == "llama-server"
and saved_backend in {"nvidia", "amd", "cpu", "intel", "sycl", "jetson"}
and saved_backend == normalize_key(args.backend)
):
image = env.get("LLAMA_SERVER_IMAGE") or image
if image and not re.fullmatch(r"[A-Za-z0-9._/@:+-]{1,300}", image):
return None
old_source = "dashboard" if state_authoritative else env.get("MODEL_SELECTION_SOURCE", "")
recommended_file = env.get("MODEL_RECOMMENDED_GGUF", "")
if host_native or old_source in {"installer", "dashboard", "operator", "preserved-local"}:
source = old_source
elif recommended_file and recommended_file != active_file:
source = "dashboard"
else:
source = "preserved-local"
contract = {
"LLM_MODEL": llm_model,
"GGUF_FILE": str(model["gguf_file"]),
"GGUF_URL": str(primary["url"]),
"GGUF_SHA256": str(primary["sha256"]),
"MAX_CONTEXT": str(context),
"LLM_MODEL_SIZE_MB": str(max(declared_mb, actual_mb)),
"MODEL_RUNTIME_PROFILE": profile_id,
"MODEL_RUNTIME_PROFILE_LABEL": str(runtime_profile.get("label") or "") if runtime_profile else "",
"MODEL_RUNTIME_PROFILE_SOURCE": str(runtime_profile.get("source_url") or "") if runtime_profile else "",
"MODEL_SELECTION_SOURCE": source,
"LLAMA_SERVER_IMAGE": image,
**runtime_values,
}
# Activation also owns this model's vision projector and fixed chat
# template. They belong to exactly this GGUF on this host's llama-server
# container: never a Windows-hosted or native macOS runtime, which take
# them as arguments, and never another model.
if (
not host_native
and "apple" not in {normalize_key(env.get("GPU_BACKEND")), normalize_key(args.backend)}
and env.get("GGUF_FILE", "").strip() == contract["GGUF_FILE"]
):
projector = carried_projector(env, model, manifest, models_dir)
if projector is not None:
contract["LLAMA_ARG_MMPROJ"] = projector
contract.update(carried_chat_template(env, getattr(args, "chat_templates", None)))
if active_store_id != "default":
contract["ODS_ACTIVE_MODEL_STORE"] = active_store_id
return contract
def context_tokens(value: str) -> int:
number = positive_int(value) if re.fullmatch(r"[0-9]{1,16}", value) else None
if number is None or number < 1024:
raise argparse.ArgumentTypeError("a whole number of tokens from 1024")
return number
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--env", type=Path, required=True)
parser.add_argument("--catalog", type=Path, required=True)
parser.add_argument("--imports", type=Path)
parser.add_argument("--models-dir", type=Path, required=True)
parser.add_argument("--state", type=Path)
parser.add_argument("--backend", default="unknown")
parser.add_argument("--memory-type", default="discrete")
parser.add_argument("--vram-mb", type=float, default=0)
parser.add_argument("--ram-gb", type=float, default=0)
parser.add_argument("--host-arch", default=platform.machine())
parser.add_argument("--local-model", action="store_true")
parser.add_argument("--chat-templates", type=Path, metavar="DIR",
help="this release's config/chat-templates; a saved fixed chat template is kept "
"only while it is there")
parser.add_argument("--native-llm", action="store_true",
help="preserve the model the ODS-owned llama-server on the Windows host serves")
parser.add_argument("--project-native-llm", metavar="MODEL_ID",
help="print the catalog record of the served model (its GGUF filename)")
parser.add_argument("--repair-native-llm", metavar="MODEL_ID",
help="re-record an installer-written mismatch from the served model (its GGUF filename)")
parser.add_argument("--served-model", metavar="MODEL_ID",
help="with --native-llm: the model the Windows host serves must be the retained one")
# Pre-round-F names, accepted for one release.
parser.add_argument("--external-lemonade", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--project-external-lemonade", metavar="MODEL_ID", help=argparse.SUPPRESS)
parser.add_argument("--repair-external-lemonade", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--context", type=context_tokens, help="the context the server loaded the model with")
args = parser.parse_args()
modes = [args.native_llm, args.external_lemonade,
args.project_native_llm is not None, args.project_external_lemonade is not None,
args.repair_native_llm is not None, args.repair_external_lemonade]
if sum(modes) > 1:
parser.error("choose one of --native-llm, --project-native-llm, --repair-native-llm")
if args.served_model is not None and not (args.native_llm or args.external_lemonade):
parser.error("--served-model applies only to --native-llm")
project_id = args.project_native_llm if args.project_native_llm is not None else args.project_external_lemonade
if project_id is not None:
contract = served_model_projection(load_records(args.catalog, args.imports), project_id, args.context)
if contract is None:
print(f"Served model {project_id!r} names no single ODS catalog model; "
"refusing to record a different model.", file=sys.stderr)
return 2
elif args.repair_native_llm is not None or args.repair_external_lemonade:
repaired = repaired_served_model_contract(args, args.repair_native_llm)
if repaired is None:
print("The saved model settings are not the installer-written mismatch "
"this release repairs; refusing to change them.", file=sys.stderr)
return 2
env, served_id, contract = repaired
changes = ", ".join(
f"{key} {env.get(key, '')} -> {contract[key]}"
for key in ("LLM_MODEL", "GGUF_FILE", "MAX_CONTEXT")
if env.get(key, "") != contract[key]
)
print(f"Repairing the saved description of the served model ({served_id}): "
f"{changes}. The served model is unchanged.", file=sys.stderr)
elif not (args.native_llm or args.external_lemonade) and is_vllm_runtime(parse_dotenv(args.env)):
contract = preserved_vllm_contract(args)
if contract is None:
# Falling through to the recommendation would turn the vLLM
# install into a llama.cpp one and download a GGUF.
print("This installation runs its model on the vLLM engine, but its saved model is not "
"a complete vLLM catalog snapshot for the GPU it names; refusing to replace it. "
"Run the installer again and choose the vLLM model, or use --reselect-model.",
file=sys.stderr)
return 2
else:
contract = preserved_contract(args)
if contract is None:
env = parse_dotenv(args.env)
if (args.native_llm or args.external_lemonade) and is_retained_external_lemonade(env):
print("Invalid retained external Lemonade model contract; refusing to replace it.", file=sys.stderr)
return 2
if (args.native_llm or args.external_lemonade) and is_retained_host_native(env):
print("Invalid retained host-native model contract, or the Windows host serves another "
"model; refusing to replace it.", file=sys.stderr)
return 2
return 0
for key, value in contract.items():
# Compose's list-form environment entries inherit exported host values.
# Emitting an empty optional LLAMA_* key would therefore turn "unset"
# into an explicit empty string inside llama.cpp, where numeric options
# such as LLAMA_ARG_N_CPU_MOE fail to parse. The installer clears stale
# selector variables before loading this fragment, so omission is the
# correct representation of an inactive optional setting.
if not value and (key in RUNTIME_KEYS or key == "LLAMA_SERVER_IMAGE"):
continue
print(f"{key}={shell_value(value)}")
return 0
if __name__ == "__main__":
sys.exit(main())