The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Per-model preset generation (--models-preset INI) — the router-side carrier for context-policy
launch decisions.
"""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass, replace
from pathlib import Path
from hermes_cli.local_runtime.context_policy import (
RUNTIME_OVERHEAD_BYTES, fit_to_free_memory, launch_args, plan_launch, ub_logits_bytes)
from hermes_cli.local_runtime.estimator import (
HardwareBudget, PhysicsRefusal, as_loaded, ctx_bytes, footprint_bytes, profile_from_gguf)
from hermes_cli.local_runtime.gguf import model_id_from_stem, read_gguf_header
logger = logging.getLogger(__name__)
# args list -> INI keys. Flags the policy owns; everything else stays out of the preset.
_FLAG_TO_KEY = {
"-c": "ctx-size", "-b": "batch-size", "-ub": "ubatch-size",
"-ctk": "cache-type-k", "-ctv": "cache-type-v", "-fa": "flash-attn",
"-ot": "override-tensor", "--spec-type": "spec-type", "--spec-draft-n-max": "spec-draft-n-max",
}
@dataclass
class PresetEntry:
model_id: str
window: int
spilled: bool
refusal: str | None = None
keys: dict[str, str] | None = None
def _args_to_keys(args: list[str]) -> dict[str, str]:
keys: dict[str, str] = {}
i = 0
while i < len(args):
key = _FLAG_TO_KEY.get(args[i])
if key is None:
i += 1
continue
keys[key] = args[i + 1]
i += 2
return keys
def _asset_path(asset) -> Path | None:
"""On-disk path of a catalog companion asset, or None when it isn't downloaded."""
from hermes_cli.local_runtime.bootstrap import assets_dir
if asset is None:
return None
path = assets_dir() / asset.local_name
return path if path.exists() else None
@dataclass(frozen=True)
class _Companions:
"""The files a catalog entry loads beside its weights, as far as they are on disk."""
mmproj: Path | None = None
mtp_head: Path | None = None
nbytes: int = 0
def _companions(entry) -> _Companions:
if entry is None:
return _Companions()
mmproj, head = _asset_path(entry.mmproj), _asset_path(entry.mtp_head)
nbytes = sum(a.size_bytes for a, p in ((entry.mmproj, mmproj), (entry.mtp_head, head)) if p)
return _Companions(mmproj=mmproj, mtp_head=head, nbytes=nbytes)
def _runs_mtp(entry, companions: _Companions) -> bool:
"""MTP spec decode runs for a catalog entry: built-in heads, or a shipped head on disk."""
return entry is not None and (entry.mtp or companions.mtp_head is not None)
def _draft_fits(path: Path, profile, budget: HardwareBudget, window: int, overhead: int) -> bool:
"""Optional draft never shrinks the advertised window or displaces its GPU buffers.
The catalog does not know the draft's context layout. Admit it only after reading the file;
draft KV defaults to f16, independently of the target's q8 cache.
"""
try:
draft = profile_from_gguf(read_gguf_header(path))
draft_need = footprint_bytes(
draft, window, flash_attention=False,
overhead_bytes=RUNTIME_OVERHEAD_BYTES + ub_logits_bytes(draft.n_vocab, mtp_capable=False))
except (ValueError, OSError) as exc:
logger.warning("draft omitted %s: %s", path.name, exc)
return False
return (footprint_bytes(profile, window, overhead_bytes=overhead) + draft_need
<= budget.usable_vram_bytes + budget.ram_available_bytes
and ctx_bytes(profile, window) + overhead + draft_need <= budget.usable_vram_bytes)
# The draft model's own context beside the target's: its KV and compute buffers. Without it the
# price below still covered every measured server (b11671, -ub 2048, one slot) by 0.2 GiB
# (Nemotron 3.5 Bolt) to 2.7 GiB (Qwen3.8 27B); this keeps the thinnest case clear.
_DRAFT_CONTEXT_BYTES = 512 << 20
def _engine_build(tag: str | None) -> int | None:
try:
return int(str(tag).removeprefix("b")) if tag else None
except ValueError:
return None
def _dflash_extra_bytes(entry, profile) -> int:
"""Fixed bytes a one-slot DFlash draft adds beside its target: the draft's weights, a copy of
the recurrent state per draft position, and the draft's own context."""
copies = entry.draft_depth + 1
return entry.draft.size_bytes + profile.recurrent_state_bytes * copies + _DRAFT_CONTEXT_BYTES
def _runs_dflash(entry) -> Path | None:
"""The entry's DFlash draft on disk, or None."""
if entry is None or entry.draft is None or entry.draft_spec_type != "draft-dflash":
return None
return _asset_path(entry.draft)
def _dflash_plan(entry, profile, budget: HardwareBudget, live: HardwareBudget | None, *,
fixed_overhead: int, requested_window: int | None, engine_tag: str | None,
window_without: int, residents: int = 1):
"""The launch plan with the entry's DFlash draft in place of MTP, or None.
The draft runs on one slot: each slot keeps a recurrent-state copy per draft position, and
the router's default four slots hold enough copies to push weights off the GPU. It may
shrink the window to half native, never below what the model gets without it when that is
smaller; a requested (grown) window it can't hold keeps the model as it was. On one slot a
side request takes the chat's slot and the prompt cache brings the chat back, so where free
commit sizes that cache the draft keeps its smallest useful size.
"""
if _runs_dflash(entry) is None:
return None
build = _engine_build(engine_tag)
if build is None or build < (_engine_build(entry.draft_min_engine) or 0):
return None
lean = replace(profile, kv_scale=1.0)
overhead = fixed_overhead + _dflash_extra_bytes(entry, lean)
plan = plan_launch(lean, budget, fixed_overhead=overhead, requested_window=requested_window)
if live is not None and live.commit_room_bytes is not None:
from hermes_cli.local_runtime.hardware import usable_beside_prompt_cache
live = replace(live, usable_vram_bytes=usable_beside_prompt_cache(live, residents))
if live is not None:
# fit_to_free_memory holds a too-busy card at the floor rather than move weights off it on a
# live reading. The draft never rides on that hold: it runs only where it fits now.
now = plan_launch(lean, live, fixed_overhead=overhead)
if isinstance(now.decision, PhysicsRefusal) or now.decision.spilled:
return None
plan = fit_to_free_memory(plan, lean, live, mtp_capable=False, fixed_overhead=overhead)
decision = plan.decision
if isinstance(decision, PhysicsRefusal) or decision.spilled:
return None
native = profile.n_ctx_train or decision.window
if requested_window and decision.window < min(requested_window, native):
return None
if decision.window < min(window_without, native // 2):
return None
return plan
def preset_for_model(gguf: Path, budget: HardwareBudget,
mtp_capable: set[str], *, requested_window: int | None = None,
live: HardwareBudget | None = None, residents: int = 1,
engine_tag: str | None = None) -> PresetEntry | None:
"""The launch decision for one staged model, or None when its header is unreadable.
``budget`` is the card's capacity; ``live`` (from ``hardware.launch_budget``) narrows the
window to what fits beside other programs now. ``residents`` is how many models the router may
hold at once; on Windows unified memory their prompt caches share the commit they leave.
``engine_tag`` is the installed llama.cpp release; without it no draft that needs a minimum
release runs.
"""
from hermes_cli.local_runtime.catalog import entry_for_model
from hermes_cli.local_runtime.growth import load_window_overrides
model_id = model_id_from_stem(gguf.stem)
try:
header = read_gguf_header(gguf)
profile = as_loaded(profile_from_gguf(header), budget)
except (ValueError, OSError) as exc:
logger.warning("preset skip %s: %s", gguf.name, exc)
return None
entry = entry_for_model(model_id)
companions = _companions(entry)
is_mtp = _runs_mtp(entry, companions) if entry is not None else model_id in mtp_capable
fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes
requested = load_window_overrides().get(model_id) if requested_window is None else requested_window
plan = plan_launch(profile, budget, mtp_capable=is_mtp, fixed_overhead=fixed_overhead,
requested_window=requested)
if live is not None:
plan = fit_to_free_memory(plan, profile, live, mtp_capable=is_mtp,
fixed_overhead=fixed_overhead)
decision = plan.decision
if isinstance(decision, PhysicsRefusal):
return PresetEntry(model_id=model_id, window=0, spilled=False, refusal=decision.message)
dflash = None if decision.spilled else _dflash_plan(
entry, profile, budget, live, fixed_overhead=fixed_overhead, requested_window=requested,
engine_tag=engine_tag, window_without=decision.window, residents=residents)
if dflash is not None:
plan, decision = dflash, dflash.decision
# Router discovery is preset-only: refused files must never autoload with stock fit.
keys = _args_to_keys(launch_args(
profile, decision, mtp_capable=is_mtp and dflash is None, uma=budget.uma,
mtp_prefill=plan.mtp_prefill,
mtp_draft_depth=entry.mtp_draft_depth if entry is not None else 3))
keys["model"] = str(gguf)
from hermes_cli.local_runtime.hardware import prompt_cache_bytes
cache = prompt_cache_bytes(live, footprint_bytes(plan.profile, decision.window,
overhead_bytes=plan.overhead_bytes), residents)
if cache is not None:
keys["cache-ram"] = str(cache >> 20) # MiB; 0 turns the cache off
if profile.lazy_bytes:
# The price above leaves these tensors on disk; llama.cpp's own default loads them on
# integrated GPUs.
keys["lazy-mode"] = "on"
if entry is not None and is_mtp:
# Integrated-MTP targets sample on the backend, and so does the draft (pairing validated
# against the vendor's published llama.cpp recipes).
keys["backend-sampling"] = "on"
keys["spec-draft-backend-sampling"] = "on"
# Sampling deference ladder, under the policy keys (policy wins on clash): the GGUF's own
# general.sampling.* metadata is the publisher's recommendation and covers models the catalog
# has never heard of; catalog sampling applies only where the file is silent; a model
# carrying neither runs llama.cpp defaults.
for k, v in header.sampling_defaults.items():
keys.setdefault(k, v)
if entry is not None:
for k, v in (entry.sampling or {}).items():
keys.setdefault(k, v)
if companions.mmproj is not None:
keys["mmproj"] = str(companions.mmproj)
if companions.mtp_head is not None:
# The model carries no MTP layers: draft-mtp drafts from the shipped head instead.
keys["model-draft"] = str(companions.mtp_head)
if dflash is not None:
keys["model-draft"] = str(_runs_dflash(entry))
keys["spec-type"] = "draft-dflash"
keys["spec-draft-n-max"] = str(entry.draft_depth)
keys["parallel"] = "1"
draft_path = (_asset_path(entry.draft) if decision.spilled
and entry.draft_spec_type == "draft-dspark" else None)
if draft_path is not None and _draft_fits(draft_path, plan.profile, budget, decision.window,
plan.overhead_bytes):
keys["model-draft"] = str(draft_path)
keys["spec-type"] = "draft-dspark"
# Unsloth's measured cliff: acceptance 83% at 2-3 drafts, collapses at 4.
keys["spec-draft-n-max"] = "3"
return PresetEntry(model_id=model_id, window=decision.window,
spilled=decision.spilled, keys=keys)
def resident_footprint(gguf: Path, budget: HardwareBudget, window: int, *, drafting: bool = False) -> int | None:
"""Estimated bytes one staged model holds while loaded at ``window``, or None when unreadable.
``drafting`` prices the one-slot DFlash launch its preset runs instead of MTP."""
from hermes_cli.local_runtime.catalog import entry_for_model
model_id = model_id_from_stem(gguf.stem)
try:
profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget)
except (ValueError, OSError) as exc:
logger.debug("footprint skip %s: %s", gguf.name, exc)
return None
entry = entry_for_model(model_id)
companions = _companions(entry)
fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes
if drafting and _runs_dflash(entry) is not None:
lean = replace(profile, kv_scale=1.0)
plan = plan_launch(lean, budget, fixed_overhead=fixed_overhead + _dflash_extra_bytes(entry, lean),
requested_window=window)
else:
plan = plan_launch(profile, budget, mtp_capable=_runs_mtp(entry, companions),
fixed_overhead=fixed_overhead, requested_window=window)
return footprint_bytes(plan.profile, window, overhead_bytes=plan.overhead_bytes)
def _launch_footprint(gguf: Path, budget: HardwareBudget, engine_tag: str | None = None) -> int | None:
"""Estimated resident bytes for one staged model at the window this policy grants it, or None
when it cannot be priced: an unreadable header, or a model the physics check refuses outright
(it never loads, so it must not shrink the residency cap)."""
from hermes_cli.local_runtime.catalog import entry_for_model
from hermes_cli.local_runtime.growth import load_window_overrides
model_id = model_id_from_stem(gguf.stem)
try:
profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget)
except (ValueError, OSError) as exc:
logger.debug("footprint skip %s: %s", gguf.name, exc)
return None
entry = entry_for_model(model_id)
companions = _companions(entry)
fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes
requested = load_window_overrides().get(model_id)
plan = plan_launch(profile, budget, mtp_capable=_runs_mtp(entry, companions),
fixed_overhead=fixed_overhead, requested_window=requested)
if isinstance(plan.decision, PhysicsRefusal):
return None
# Priced whole even when the plan spills: a spilled model still holds part of its weights on
# the device, and over-counting errs toward the side that cannot thrash.
price = footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes)
# Where the engine loads the draft, the preset runs it in place of MTP: the model holds the
# larger of the two launches, never both.
dflash = None if plan.decision.spilled else _dflash_plan(
entry, profile, budget, None, fixed_overhead=fixed_overhead, requested_window=requested,
engine_tag=engine_tag, window_without=plan.decision.window)
if dflash is not None:
price = max(price, footprint_bytes(dflash.profile, dflash.decision.window,
overhead_bytes=dflash.overhead_bytes))
return price
def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int,
live: HardwareBudget | None = None, engine_tag: str | None = None) -> int:
"""How many models the card may hold resident at once: priced against the budget, not a count.
Residency used to be bounded by a count alone, so a second model was admitted against an
already-full card. On Windows/WDDM that over-commit is not refused — the allocation is paged
to host memory, and that child decodes at a third of its speed for the rest of its life: no
error, no UI hint, and ejecting the incumbent afterwards does not repair it (only a clean
reload does). Capping the count instead has llama.cpp evict its LRU *before* the incoming
child allocates, which is the only placement that fits.
The cap rises above one only while the LARGEST staged model still fits TWICE — any pair of
staged models then fits by construction. ``configured`` stays a ceiling (a user's smaller
number is honoured), and an unpriceable input (no usable device memory, no readable model)
keeps today's behaviour. On Windows unified memory ``live`` (free commit) bounds it too: the
models the router holds together must fit in commit, not just in the pool.
"""
from hermes_cli.local_runtime.bootstrap import staged_in
if configured <= 1 or budget.usable_vram_bytes <= 0:
return configured
largest = 0
for gguf in staged_in(models_dir):
need = _launch_footprint(gguf, budget, engine_tag)
if need:
largest = max(largest, need)
if largest <= 0:
return configured
usable = budget.usable_vram_bytes
if live is not None and live.commit_room_bytes is not None:
usable = min(usable, live.usable_vram_bytes)
return max(1, min(configured, usable // largest))
def plan_presets(models_dir: Path, budget: HardwareBudget, mtp_capable: set[str] | None = None,
*, live: HardwareBudget | None = None, residents: int = 1,
engine_tag: str | None = None) -> list[PresetEntry]:
"""The launch decision for every staged model; unreadable headers are skipped."""
from hermes_cli.local_runtime.bootstrap import staged_in
entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, residents=residents,
engine_tag=engine_tag)
for gguf in staged_in(models_dir))
return [entry for entry in entries if entry is not None]
def generate_presets(models_dir: Path, budget: HardwareBudget, preset_path: Path,
mtp_capable: set[str] | None = None, *,
live: HardwareBudget | None = None, residents: int = 1,
engine_tag: str | None = None) -> list[PresetEntry]:
"""Plan every staged model and write one INI. Refused models get no section (the picker
surfaces the refusal from the returned entries)."""
entries = plan_presets(models_dir, budget, mtp_capable, live=live, residents=residents,
engine_tag=engine_tag)
write_presets(entries, preset_path)
return entries
def write_presets(entries: list[PresetEntry], preset_path: Path) -> None:
from utils import atomic_write_text
atomic_write_text(preset_path, render_presets(entries), tmp_prefix=f".{preset_path.name}_", mode=0o600)
logger.info("wrote %d preset sections to %s", sum(e.keys is not None for e in entries), preset_path)
def render_presets(entries: list[PresetEntry]) -> str:
sections: list[str] = []
for entry in entries:
# INI comments preserve non-flag facts atomically with the launch policy.
sections.append("# hermes-decision: " + json.dumps({
"model_id": entry.model_id, "window": entry.window,
"spilled": entry.spilled, "refusal": entry.refusal}) + "\n")
if entry.keys is not None:
body = "\n".join(f"{k} = {v}" for k, v in entry.keys.items())
sections.append(f"[{entry.model_id}]\n{body}\n")
return "\n".join(sections)
def read_preset_decisions(preset_path: Path | None = None) -> dict[str, PresetEntry]:
"""The launch decisions the running server was actually given, read back from the preset INI
(the INI is the record — it's what spawned the children). Missing/unparseable -> {}."""
import configparser
if preset_path is None:
from hermes_cli.local_runtime.binaries import runtimes_root
preset_path = runtimes_root() / "presets.ini"
out: dict[str, PresetEntry] = {}
try:
parser = configparser.ConfigParser(interpolation=None)
text = preset_path.read_text(encoding="utf-8-sig")
parser.read_string(text)
recorded = {}
for line in text.splitlines():
if line.startswith("# hermes-decision: "):
fact = json.loads(line.removeprefix("# hermes-decision: "))
recorded[fact["model_id"]] = fact
if fact.get("refusal"):
out[fact["model_id"]] = PresetEntry(**fact)
for section in parser.sections():
out[section] = PresetEntry(
model_id=section, window=parser.getint(section, "ctx-size", fallback=0),
spilled=recorded.get(section, {}).get("spilled", parser.has_option(section, "override-tensor")),
keys=dict(parser[section]))
except Exception as exc:
logger.debug("preset read-back failed: %s", exc)
return out