ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
"""Versioned model-state record for the ODS Model Switchboard (PR 1).
Contract highlights (see ods/docs/MODEL-SWITCHBOARD.md, section 3.2):
- The host agent is the only writer. Writes are temp-file + flush + fsync
where available + atomic replace; a reader can never observe partial JSON.
- ``seq`` increments on every mutation; ``routeSeq`` increments only when the
active route changes.
- ``active`` always remains the last proven route. In observe mode this module
is called only after the existing activation transaction has already proved
success, so a failed activation never touches the record.
- ``history`` keeps the last ``HISTORY_LIMIT`` verified active routes.
- Startup reconstruction from ``.env`` is permitted only when no v1 state has
ever been committed, and is marked ``reconstructed`` with an unproven
completion so it can never masquerade as a verified proof.
Stdlib only: the standalone host agent imports this from the installed tree.
"""
from __future__ import annotations
import json
import os
import tempfile
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
SCHEMA_VERSION = "ods.model-state.v1"
HISTORY_LIMIT = 10
PUBLIC_MODEL_DEFAULT = "ods/current"
STATE_FILE_MODE = 0o644
DEFAULT_ENDPOINT_ID = "llama-server-default"
VLLM_ENDPOINT_ID = "vllm-default"
# Every writable backend kind and the model-router endpoint (endpoints.json
# id) it routes to when the caller names none. hipfire and unknown routes
# have always used the llama-server endpoint; vLLM serves from vllm-server.
_DEFAULT_ENDPOINT_IDS = {
"llama-server": DEFAULT_ENDPOINT_ID,
"hipfire": DEFAULT_ENDPOINT_ID,
"unknown": DEFAULT_ENDPOINT_ID,
"vllm": VLLM_ENDPOINT_ID,
}
_WRITABLE_BACKEND_KINDS = set(_DEFAULT_ENDPOINT_IDS)
# Readable for one release only: records written before round F carry the
# retired Lemonade route until the host agent rewrites and re-proves them.
_LEGACY_BACKEND_KINDS = {"lemonade"}
_LEGACY_ENDPOINT_IDS = {"lemonade-default"}
_BACKEND_KINDS = _WRITABLE_BACKEND_KINDS | _LEGACY_BACKEND_KINDS
_OPERATION_PHASES = {
"requested", "staging", "verifying", "publishing",
"flipping", "serving", "failed", "rolling_back",
}
_AVAILABILITY_MODES = {"serve_active", "queue"}
_TOP_LEVEL_KEYS = {
"schema", "seq", "routeSeq", "operation", "desired", "active",
"history", "availability",
}
_OPERATION_KEYS = {"id", "phase", "requestedModelId", "startedAt", "error"}
_ACTIVE_KEYS = {
"routeSeq", "catalogId", "runtimeModelId", "publicModel", "backend",
"contextLength", "capabilities", "verifiedAt", "reconstructed", "proof",
}
_BACKEND_KEYS = {"kind", "endpointId", "nativeRoute"}
_CAPABILITY_KEYS = {"chat", "tools", "vision", "agentViable"}
_PROOF_KEYS = {"identity", "completion"}
_HISTORY_KEYS = {"routeSeq", "catalogId", "runtimeModelId", "verifiedAt"}
_AVAILABILITY_KEYS = {"mode", "queueDeadline"}
_WRITE_LOCK = threading.Lock()
_LAST_GOOD: dict[str, dict[str, Any]] = {}
class StateError(RuntimeError):
"""Raised for unrecoverable state-record violations (writer side)."""
def _utcnow_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _check_keys(
value: dict[str, Any],
*,
allowed: set[str] | None,
required: set[str],
label: str,
errors: list[str],
) -> None:
missing = required - set(value)
unexpected = set(value) - allowed if allowed is not None else set()
if missing:
errors.append(f"{label} is missing required keys: {', '.join(sorted(missing))}")
if unexpected:
errors.append(f"{label} has unexpected keys: {', '.join(sorted(unexpected))}")
def initial_state() -> dict[str, Any]:
return {
"schema": SCHEMA_VERSION,
"seq": 0,
"routeSeq": 0,
"operation": None,
"desired": None,
"active": None,
"history": [],
"availability": {"mode": "serve_active", "queueDeadline": None},
}
def validate_state(doc: Any) -> list[str]:
"""Structural validation without third-party dependencies.
Returns a list of human-readable problems; empty means valid. The JSON
Schema at ods/config/model-state.schema.v1.json is the authoritative
published contract; this validator must stay in agreement with it.
"""
errors: list[str] = []
if not isinstance(doc, dict):
return ["state root must be an object"]
_check_keys(
doc,
allowed=_TOP_LEVEL_KEYS,
required=_TOP_LEVEL_KEYS - {"operation"},
label="state",
errors=errors,
)
if doc.get("schema") != SCHEMA_VERSION:
errors.append(f"schema must be {SCHEMA_VERSION!r}")
for key in ("seq", "routeSeq"):
value = doc.get(key)
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
errors.append(f"{key} must be a non-negative integer")
operation = doc.get("operation")
if operation is not None:
if not isinstance(operation, dict):
errors.append("operation must be null or an object")
else:
_check_keys(
operation,
allowed=_OPERATION_KEYS,
required={"id", "phase", "requestedModelId", "startedAt"},
label="operation",
errors=errors,
)
if not isinstance(operation.get("id"), str) or not operation.get("id"):
errors.append("operation.id must be a non-empty string")
if operation.get("phase") not in _OPERATION_PHASES:
errors.append("operation.phase is not a known phase")
if not isinstance(operation.get("requestedModelId"), str):
errors.append("operation.requestedModelId must be a string")
if not isinstance(operation.get("startedAt"), str):
errors.append("operation.startedAt must be a string")
if operation.get("error") is not None and not isinstance(
operation.get("error"), str
):
errors.append("operation.error must be a string or null")
desired = doc.get("desired")
if desired is not None:
if not isinstance(desired, dict):
errors.append("desired must be null or {catalogId: non-empty string}")
else:
_check_keys(
desired,
allowed={"catalogId"},
required={"catalogId"},
label="desired",
errors=errors,
)
if (
not isinstance(desired.get("catalogId"), str)
or not desired.get("catalogId")
):
errors.append("desired.catalogId must be a non-empty string")
active = doc.get("active")
if active is not None:
if not isinstance(active, dict):
errors.append("active must be null or an object")
else:
_check_keys(
active,
allowed=_ACTIVE_KEYS,
required=_ACTIVE_KEYS - {"reconstructed"},
label="active",
errors=errors,
)
active_route_seq = active.get("routeSeq")
if (
not isinstance(active_route_seq, int)
or isinstance(active_route_seq, bool)
or active_route_seq < 0
):
errors.append("active.routeSeq must be a non-negative integer")
for key in ("catalogId", "runtimeModelId", "publicModel"):
if not isinstance(active.get(key), str) or not active.get(key):
errors.append(f"active.{key} must be a non-empty string")
backend = active.get("backend")
if not isinstance(backend, dict):
errors.append("active.backend must be an object")
else:
_check_keys(
backend,
allowed=_BACKEND_KEYS,
required={"kind", "endpointId"},
label="active.backend",
errors=errors,
)
if backend.get("kind") not in _BACKEND_KINDS:
errors.append("active.backend.kind is not a known backend kind")
if (
not isinstance(backend.get("endpointId"), str)
or not backend.get("endpointId")
):
errors.append("active.backend.endpointId must be a non-empty string")
native_route = backend.get("nativeRoute")
if native_route is not None and not isinstance(native_route, str):
errors.append("active.backend.nativeRoute must be a string or null")
context_length = active.get("contextLength")
if not isinstance(context_length, int) or isinstance(context_length, bool) or context_length < 0:
errors.append("active.contextLength must be a non-negative integer")
capabilities = active.get("capabilities")
if not isinstance(capabilities, dict):
errors.append("active.capabilities must be an object")
else:
_check_keys(
capabilities,
allowed=_CAPABILITY_KEYS,
required=_CAPABILITY_KEYS,
label="active.capabilities",
errors=errors,
)
for key in ("chat", "tools", "vision", "agentViable"):
if not isinstance(capabilities.get(key), bool):
errors.append(f"active.capabilities.{key} must be a boolean")
verified_at = active.get("verifiedAt")
if verified_at is not None and not isinstance(verified_at, str):
errors.append("active.verifiedAt must be a string or null")
if "reconstructed" in active and not isinstance(active.get("reconstructed"), bool):
errors.append("active.reconstructed must be a boolean")
proof = active.get("proof")
if not isinstance(proof, dict):
errors.append("active.proof must be {identity, completion: bool}")
else:
_check_keys(
proof,
allowed=_PROOF_KEYS,
required=_PROOF_KEYS,
label="active.proof",
errors=errors,
)
if proof.get("identity") is not None and not isinstance(
proof.get("identity"), str
):
errors.append("active.proof.identity must be a string or null")
if not isinstance(proof.get("completion"), bool):
errors.append("active.proof.completion must be a boolean")
history = doc.get("history")
if not isinstance(history, list):
errors.append("history must be an array")
else:
if len(history) > HISTORY_LIMIT:
errors.append(f"history exceeds {HISTORY_LIMIT} entries")
for index, entry in enumerate(history):
if not isinstance(entry, dict):
errors.append(f"history[{index}] must be an object")
continue
_check_keys(
entry,
allowed=None,
required=_HISTORY_KEYS,
label=f"history[{index}]",
errors=errors,
)
history_route_seq = entry.get("routeSeq")
if (
not isinstance(history_route_seq, int)
or isinstance(history_route_seq, bool)
or history_route_seq < 0
):
errors.append(f"history[{index}].routeSeq must be a non-negative integer")
for key in ("catalogId", "runtimeModelId"):
if not isinstance(entry.get(key), str):
errors.append(f"history[{index}].{key} must be a string")
if entry.get("verifiedAt") is not None and not isinstance(
entry.get("verifiedAt"), str
):
errors.append(f"history[{index}].verifiedAt must be a string or null")
availability = doc.get("availability")
if not isinstance(availability, dict):
errors.append("availability.mode must be serve_active or queue")
else:
_check_keys(
availability,
allowed=_AVAILABILITY_KEYS,
required=_AVAILABILITY_KEYS,
label="availability",
errors=errors,
)
if availability.get("mode") not in _AVAILABILITY_MODES:
errors.append("availability.mode must be serve_active or queue")
if availability.get("queueDeadline") is not None and not isinstance(
availability.get("queueDeadline"), str
):
errors.append("availability.queueDeadline must be a string or null")
return errors
def read_state(path: os.PathLike | str) -> tuple[dict[str, Any] | None, list[str]]:
"""Read and validate the state record.
Returns ``(doc, [])`` on success. On a missing file returns
``(None, [])``. On malformed or invalid content returns
``(last_known_good_or_None, errors)`` — malformed state never promotes
anything and never raises out of a reader.
"""
key = str(Path(path))
raw = None
last_exc: OSError | None = None
for _attempt in range(4):
try:
raw = Path(path).read_text(encoding="utf-8")
break
except FileNotFoundError:
return None, []
except PermissionError as exc:
# Windows: os.replace briefly locks the destination; a concurrent
# reader can hit a transient sharing violation. Retry, then fall
# back to the last known-good snapshot.
last_exc = exc
time.sleep(0.005)
except OSError as exc:
return _LAST_GOOD.get(key), [f"state read failed: {exc}"]
if raw is None:
return _LAST_GOOD.get(key), [f"state read failed: {last_exc}"]
try:
doc = json.loads(raw)
except ValueError as exc:
return _LAST_GOOD.get(key), [f"state is not valid JSON: {exc}"]
errors = validate_state(doc)
if errors:
return _LAST_GOOD.get(key), errors
_LAST_GOOD[key] = doc
return doc, []
def atomic_write_state(path: os.PathLike | str, doc: dict[str, Any]) -> None:
"""Atomically persist ``doc``: same-directory temp file, fsync, replace."""
errors = validate_state(doc)
if errors:
raise StateError("refusing to write invalid state: " + "; ".join(errors))
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
payload = json.dumps(doc, indent=2, sort_keys=False) + "\n"
fd, tmp_name = tempfile.mkstemp(
prefix=target.name + ".", suffix=".tmp", dir=str(target.parent)
)
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
handle.write(payload)
handle.flush()
try:
os.fsync(handle.fileno())
except OSError:
pass
try:
os.chmod(tmp_name, STATE_FILE_MODE)
except OSError:
pass
replace_error: OSError | None = None
for _attempt in range(40):
try:
os.replace(tmp_name, target)
replace_error = None
break
except PermissionError as exc:
# Windows: a concurrent reader holding the destination open can
# make the atomic rename transiently fail with a sharing
# violation. Retry briefly; the writer is the single mutator.
replace_error = exc
time.sleep(0.005)
if replace_error is not None:
raise replace_error
except BaseException:
try:
os.unlink(tmp_name)
except OSError:
pass
raise
try: # best-effort directory durability on POSIX
dir_fd = os.open(str(target.parent), getattr(os, "O_DIRECTORY", os.O_RDONLY))
try:
os.fsync(dir_fd)
finally:
os.close(dir_fd)
except OSError:
pass
_LAST_GOOD[str(target)] = doc
def record_verified_route(
path: os.PathLike | str,
*,
catalog_id: str,
runtime_model_id: str,
backend_kind: str,
endpoint_id: str,
context_length: int,
capabilities: dict[str, bool],
proof_identity: str | None,
proof_completion: bool = True,
native_route: str | None = None,
public_model: str = PUBLIC_MODEL_DEFAULT,
reconstructed: bool = False,
) -> dict[str, Any]:
"""Record a proven active route (observe mode: call only after success).
Reload-modify-write under a process lock. If the on-disk record is valid
and newer than our cache, the on-disk record wins as the base — a writer
can never regress ``seq``.
"""
if backend_kind in _LEGACY_BACKEND_KINDS or endpoint_id in _LEGACY_ENDPOINT_IDS:
raise StateError(f"legacy route {backend_kind}/{endpoint_id} is readable only")
if backend_kind not in _WRITABLE_BACKEND_KINDS:
backend_kind = "unknown"
with _WRITE_LOCK:
base, errors = read_state(path)
if base is None:
if errors:
# Malformed state on disk: do not guess; start a fresh record
# but never delete the malformed file's information silently.
raise StateError(
"existing state is malformed; refusing blind overwrite: "
+ "; ".join(errors)
)
base = initial_state()
doc = json.loads(json.dumps(base)) # deep copy, stdlib only
previous = doc.get("active")
route_changed = not (
isinstance(previous, dict)
and previous.get("runtimeModelId") == runtime_model_id
and isinstance(previous.get("backend"), dict)
and previous["backend"].get("kind") == backend_kind
and previous["backend"].get("endpointId") == endpoint_id
)
doc["seq"] = int(doc["seq"]) + 1
if route_changed:
doc["routeSeq"] = int(doc["routeSeq"]) + 1
previous_proof = previous.get("proof") if isinstance(previous, dict) else None
previous_is_verified = bool(
isinstance(previous, dict)
and isinstance(previous_proof, dict)
and previous_proof.get("completion") is True
and isinstance(previous.get("verifiedAt"), str)
and previous.get("verifiedAt")
and previous.get("reconstructed") is not True
)
if previous_is_verified:
history_entry = {
"routeSeq": previous.get("routeSeq", 0),
"catalogId": previous.get("catalogId", ""),
"runtimeModelId": previous.get("runtimeModelId", ""),
"verifiedAt": previous.get("verifiedAt"),
}
doc["history"] = ([history_entry] + list(doc.get("history") or []))[:HISTORY_LIMIT]
doc["desired"] = {"catalogId": catalog_id}
active: dict[str, Any] = {
"routeSeq": doc["routeSeq"],
"catalogId": catalog_id,
"runtimeModelId": runtime_model_id,
"publicModel": public_model,
"backend": {
"kind": backend_kind,
"endpointId": endpoint_id,
"nativeRoute": native_route,
},
"contextLength": int(context_length),
"capabilities": {
"chat": bool(capabilities.get("chat", True)),
"tools": bool(capabilities.get("tools", False)),
"vision": bool(capabilities.get("vision", False)),
"agentViable": bool(capabilities.get("agentViable", False)),
},
"verifiedAt": None if reconstructed else _utcnow_iso(),
"proof": {"identity": proof_identity, "completion": bool(proof_completion)},
}
if reconstructed:
active["reconstructed"] = True
doc["active"] = active
doc["operation"] = None
doc["availability"] = {"mode": "serve_active", "queueDeadline": None}
atomic_write_state(path, doc)
return doc
def is_legacy_route(active: Any) -> bool:
"""Whether ``active`` still names the retired Lemonade route (one release)."""
backend = active.get("backend") if isinstance(active, dict) else None
return isinstance(backend, dict) and (
backend.get("kind") in _LEGACY_BACKEND_KINDS
or backend.get("endpointId") in _LEGACY_ENDPOINT_IDS
)
def default_endpoint_id(backend_kind: str) -> str:
"""The router endpoint id a writable backend kind uses by default."""
return _DEFAULT_ENDPOINT_IDS[backend_kind]
def migrate_env_identity(env: dict[str, str]) -> dict[str, Any] | None:
"""Derive a best-effort runtime identity from ``.env`` values.
A llama-server runtime serves ``--alias <GGUF_FILE>``, so its identity is
the GGUF filename (or the model name when no file is recorded). A vLLM
runtime (``LLM_BACKEND=vllm``) serves ``--served-model-name
<VLLM_SERVED_MODEL>`` (or the model name when none is recorded); a
``GGUF_FILE`` left by an earlier llama-server install never names it. A
retired ``LEMONADE_MODEL`` line is never read: a leftover key must not
relabel the route. Returns ``None`` when the environment carries no local
model identity (e.g. cloud-only).
"""
if str(env.get("ODS_MODE") or "").strip().casefold() == "cloud":
return None
backend = str(env.get("LLM_BACKEND") or "").strip().lower()
if backend in {"", "lemonade"}:
# A pre-migration LLM_BACKEND=lemonade install now serves through
# upstream llama-server; the retired kind is never derived again.
backend = "llama-server"
if backend not in _WRITABLE_BACKEND_KINDS:
backend = "unknown"
llm_model = str(env.get("LLM_MODEL") or "").strip()
if backend == "vllm":
served = str(env.get("VLLM_SERVED_MODEL") or "").strip()
else:
served = str(env.get("GGUF_FILE") or "").strip()
runtime_id = served or llm_model
if not runtime_id:
return None
catalog_guess = llm_model
if not catalog_guess:
stem = runtime_id
if stem.lower().endswith(".gguf"):
stem = stem[: -len(".gguf")]
catalog_guess = stem
context = 0
for key in ("CTX_SIZE", "MAX_CONTEXT"):
try:
candidate = int(str(env.get(key) or "0").strip() or 0)
except (TypeError, ValueError):
continue
if candidate > 0:
context = candidate
break
return {
"catalogId": catalog_guess,
"runtimeModelId": runtime_id,
"backendKind": backend,
"contextLength": max(0, context),
}
def initialize_if_missing(
path: os.PathLike | str,
env: dict[str, str],
*,
endpoint_id: str | None = None,
) -> dict[str, Any] | None:
"""One-time startup reconstruction when no v1 state was ever committed.
Existing files — valid or malformed — are never overwritten here. Without
an ``endpoint_id`` the route uses its backend kind's default endpoint.
Returns the written doc, or ``None`` when nothing was written.
"""
if Path(path).exists():
return None
identity = migrate_env_identity(env)
if identity is None:
return None
if endpoint_id is None:
endpoint_id = default_endpoint_id(identity["backendKind"])
return record_verified_route(
path,
catalog_id=identity["catalogId"],
runtime_model_id=identity["runtimeModelId"],
backend_kind=identity["backendKind"],
endpoint_id=endpoint_id,
context_length=identity["contextLength"],
capabilities={
"chat": True,
"tools": False,
"vision": False,
"agentViable": identity["contextLength"] >= 65536,
},
proof_identity=identity["runtimeModelId"],
proof_completion=False,
reconstructed=True,
)