Skip to content

The agent that grows with you

20.5 KB
Raw Blame
"""Curated starter catalog for the managed local runtime.

Every entry carries the estimator inputs (measured on real GGUFs) so the picker can price a model
BEFORE the user downloads gigabytes; once a file is on disk, profile_from_gguf() is the authority.
Builds proven end-to-end on real hardware are marked validated; day-0 entries ship without the flag
— ensure_model_ready's touch generation still gates every first load at runtime.
"""

from __future__ import annotations

import json
import logging
import threading
import time
import urllib.request
from dataclasses import dataclass, field
from pathlib import PurePosixPath

from hermes_cli.local_runtime.context_policy import (
    FLOOR, RUNTIME_OVERHEAD_BYTES, TARGET_WINDOW, LaunchPlan, plan_launch)
from hermes_cli.local_runtime.estimator import (
    HardwareBudget, LayerKind, ModelProfile, PhysicsRefusal, as_loaded)
from hermes_cli.local_runtime.gguf import model_id_from_stem
from hermes_platform.host.products import is_nvidia_n1x_pci_id

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class AssetFile:
    """One downloadable file: repo-relative path and exact bytes (feeds the estimator and the
    progress bar; no download-time integrity check by design — a corrupt file surfaces as a
    llama.cpp load error). ``local`` overrides the on-disk name (repos reuse generic names like
    mmproj-BF16.gguf). Non-model extras live under the models dir's assets/ subdirectory so the
    router never lists them.
    """

    path: str                   # repo-relative (may include a subdir)
    size_bytes: int
    local: str | None = None
    repo: str | None = None     # HF repo when it is not the entry's own (a companion published elsewhere)
    revision: str = "main"      # commit whose bytes size_bytes and the entry's pricing describe

    @property
    def local_name(self) -> str:
        return self.local or PurePosixPath(self.path).name


@dataclass(frozen=True)
class QuantVariant:
    """One downloadable build. Split GGUFs list every part in files; the model loads from the
    first part."""

    quant: str                  # e.g. "UD-Q4_K_M"
    files: tuple                # AssetFile, first = the load target
    validated: bool = False     # proven end-to-end on real hardware
    # Bytes of tensors the engine reads from the file on demand instead of loading (see
    # gguf._LAZY_READ_TENSORS), taken from the build's tensor table at authoring time.
    lazy_bytes: int = 0

    @property
    def model_id(self) -> str:
        return model_id_from_stem(PurePosixPath(self.files[0].path).name.removesuffix(".gguf"))

    @property
    def size_bytes(self) -> int:
        return sum(f.size_bytes for f in self.files)

    @property
    def weights_bytes(self) -> int:
        """Pre-download estimate of the weights the engine loads: GGUF bytes ≈ tensor bytes + a <2%
        header, less the tensors it reads from disk on demand — slightly conservative until
        profile_from_gguf reads the real table."""
        return self.size_bytes - self.lazy_bytes


@dataclass(frozen=True)
class CatalogEntry:
    id: str                     # stable family id (variant-independent)
    display_name: str
    description: str            # one line, plain language
    repo: str                   # HF repo
    variants: tuple             # QuantVariant (exactly one, Q4-class)
    # Estimator inputs (measured or config-derived; quant changes weights, never KV). Gated
    # upstream configs carry a conservative same-family prior — the GGUF header is the authority
    # after download.
    n_ctx_train: int
    full_layers: int
    recurrent_layers: int
    per_layer_f16: int          # KV bytes/token per full-attention layer
    swa_layers: int = 0
    swa_window: int = 0
    moe: bool = False
    mtp: bool = False           # ships MTP heads (spec decode when loaded)
    # Speculative draft depth for MTP models: per-model and measured — deeper drafting pays only
    # while draft acceptance holds, and the break-even depth differs by model.
    mtp_draft_depth: int = 3
    # Vocab size prices the GPU logits buffers (ubatch x vocab x fp32, doubled under MTP backend
    # sampling) — a multi-GiB term at large vocabs that a weights-only fit would miss.
    n_vocab: int = 0
    # GGUF general.architecture; prices architecture-specific buffers (estimator._WINDOW_COMPUTE_BYTES)
    # before the file is on disk.
    architecture: str = ""
    mmproj: AssetFile | None = None    # vision projector, downloads with model
    draft: AssetFile | None = None     # spec-decode draft model (e.g. DSpark)
    # How the draft decodes. draft-dspark drafts only for a model spilled past GPU memory;
    # draft-dflash replaces MTP whenever it fits, on one slot (each slot keeps a recurrent-state
    # copy per draft position). Optional: a missing draft leaves the model as it was.
    draft_spec_type: str = "draft-dspark"
    draft_depth: int = 3
    draft_min_engine: str = ""          # oldest llama.cpp release that loads the draft
    # MTP head shipped as its own file (the model carries none): the engine loads it as the
    # draft for MTP spec decode. Downloads with the model.
    mtp_head: AssetFile | None = None
    sampling: dict = field(default_factory=dict)  # INI long-form launch defaults
    # Oldest llama.cpp release tag that can load this model (day-0 architectures need the release
    # where their support landed). Empty means any installed engine.
    min_engine: str = ""
    # Editorial quality ordering (higher = smarter), authored once at catalog time — Artificial
    # Analysis-informed where covered (scripts/aa_quality_sync.py proposes, the commit decides).
    # Ranks entries for the per-machine recommendation; never displayed as a score (it grades the
    # full-precision model, not our Q4 build).
    quality: int = 0
    # Fraction of the build's bytes read per decoded token: 1.0 for dense, the active slice for
    # MoE. With memory bandwidth this predicts decode speed — the physics half of the
    # recommendation.
    decode_fraction: float = 1.0
    # False lists the entry for explicit choice only: it is never the automatic recommendation or
    # quickstart's pick on any machine, so it needs no quality ordering.
    recommend: bool = True

    @property
    def mtp_capable(self) -> bool:
        """MTP spec decode runs, from heads built into the model or shipped beside it."""
        return self.mtp or self.mtp_head is not None

    @property
    def companion_bytes(self) -> int:
        """Companion files the engine loads beside the weights (vision projector, MTP head)."""
        return sum(a.size_bytes for a in (self.mmproj, self.mtp_head) if a is not None)

    def profile(self, variant: QuantVariant) -> ModelProfile:
        layers = ([(LayerKind.FULL, self.per_layer_f16)] * self.full_layers
                  + [(LayerKind.SWA, self.per_layer_f16)] * self.swa_layers
                  + [(LayerKind.RECURRENT, 0)] * self.recurrent_layers)
        return ModelProfile(
            name=variant.model_id, weights_bytes=variant.weights_bytes, embd_table_bytes=0,
            n_ctx_train=self.n_ctx_train, layers=layers, swa_window=self.swa_window, moe=self.moe,
            architecture=self.architecture, n_vocab=self.n_vocab,
            kv_scale=1.2 if self.mtp_capable else 1.0, lazy_bytes=variant.lazy_bytes)

    def launch_plan(self, variant: QuantVariant, budget: HardwareBudget) -> LaunchPlan:
        # Optional external drafts may use spare memory after download, never reduce this grant.
        return plan_launch(as_loaded(self.profile(variant), budget), budget, mtp_capable=self.mtp_capable,
                           fixed_overhead=RUNTIME_OVERHEAD_BYTES + self.companion_bytes)

    def download_files(self, variant: QuantVariant) -> tuple:
        """Everything a download job fetches for this variant, in order."""
        extras = tuple(a for a in (self.mmproj, self.draft, self.mtp_head) if a is not None)
        return tuple(variant.files) + extras

    def download_bytes(self, variant: QuantVariant) -> int:
        return sum(f.size_bytes for f in self.download_files(variant))

    def install_files(self, variant: QuantVariant) -> tuple:
        """The files a variant needs to count as downloaded. The draft is optional: a model
        installed before its draft joined the catalog stays installed."""
        extras = tuple(a for a in (self.mmproj, self.mtp_head) if a is not None)
        return tuple(variant.files) + extras


@dataclass(frozen=True)
class VariantChoice:
    """Which build this machine should download and why. reason_key is a UI-copy
    discriminator, not display text."""

    variant: QuantVariant
    zero_spill: bool
    reason_key: str  # "best-large-window" | "best-fits" | "smallest-fits-spilled"


def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice | None:
    """Fit the entry's one Q4-class build to this machine; headroom buys a bigger window, never a
    bigger quant.

    "best-large-window": zero-spill at TARGET_WINDOW; "best-fits": zero-spill at the 64K floor;
    "smallest-fits-spilled": weights spill to host RAM, priced honestly; None: physics refuses.
    """
    variant = entry.variants[-1]
    decision = entry.launch_plan(variant, budget).decision
    if isinstance(decision, PhysicsRefusal):
        return None
    if decision.spilled:
        return VariantChoice(variant, zero_spill=False, reason_key="smallest-fits-spilled")
    reason = "best-large-window" if decision.window >= min(TARGET_WINDOW, entry.n_ctx_train or FLOOR) else "best-fits"
    return VariantChoice(variant, zero_spill=True, reason_key=reason)


# ── recommendation: best quality that fits and isn't miserably slow ──
#
# QUALITY is a judgment made once at authoring time (entry.quality). SPEED is physics per machine:
# decode is memory-bound, so predicted tok/s ≈ bandwidth / bytes-read-per-token (build size scaled
# by decode fraction). The bandwidth axis is the `uma` flag: every discrete card that matters is
# 900+ GB/s GDDR while the unified-memory class measures ~1/5th of that. A measured per-machine
# bandwidth could replace these class constants without touching the rule; predictions order
# candidates and gate the floor — they are not display values.

_DISCRETE_BANDWIDTH_GB_S = 1000.0   # representative GDDR6X/GDDR7 class
_UMA_BANDWIDTH_GB_S = 210.0         # measured on unified-memory NVIDIA
_HOST_BANDWIDTH_GB_S = 80.0         # spilled weights stream over host DRAM

# Below this predicted decode speed a model stops feeling pleasant for agentic use (roughly
# reading speed with headroom for tool-call bursts). Distinct from the growth policy's 6 tok/s
# compress floor, which marks unusable, not unpleasant.
PLEASANT_FLOOR_TOK_S = 20.0

# Shipped short-context reference rates, not benchmarks run during recommendation.
# Windows N1X / b10964 CUDA: MTP2 measured 21.76–21.96 tok/s over four 512-token
# prose probes; a four-slot smoke measured 21.89. At 32K input the reference was
# 18.82 tok/s: this is a baseline estimate, not a context-independent guarantee.
# Unmatched hardware, backend, quant or draft depth retains the bandwidth estimate.
_MEASURED_DECODE_TOK_S = {
    ("win32", "cuda", "NVIDIA RTX Spark N1X",
     "qwen3.8-27b", "UD-Q4_K_M", 2): 21.9,
}


# Defaults the maker of a recognized product chose for it, keyed like the measured rates. The
# derived rule still decides when the chosen entry is ineligible or cannot run resident, and every
# other entry that fits stays available. The RTX Spark ships the 27B: Flash Next fits 128 GB, but
# Windows backs GPU memory with commit outside the carve-out, so it loads only with the carve-out
# lowered.
_PRODUCT_DEFAULTS = {
    ("win32", "cuda", "NVIDIA RTX Spark N1X"): "qwen3.8-27b",
}


def _hardware_key(budget: HardwareBudget, backend: str) -> tuple[str, str, str]:
    """(platform, backend, reference GPU name) for the shipped tables; the name is empty when the
    GPU is not recognized."""
    # Drivers may append a parenthesized description to the stable device name.
    gpu_name = budget.gpu_name.partition(" (")[0]
    # Resolve PCI identity to the existing reference key; names only backfill missing IDs.
    if budget.gpu_pci_id is not None:
        gpu_name = "NVIDIA RTX Spark N1X" if is_nvidia_n1x_pci_id(budget.gpu_pci_id) else ""
    effective_backend = "cuda" if backend == "auto" and gpu_name else backend
    return (budget.platform, effective_backend, gpu_name)


def predicted_decode_tok_s(entry: CatalogEntry, variant: QuantVariant, budget: HardwareBudget, *,
                           spilled: bool = False, backend: str = "auto") -> float:
    """Shipped measured baseline where matched, otherwise the memory-bound estimate."""
    key = (*_hardware_key(budget, backend), entry.id, variant.quant, entry.mtp_draft_depth)
    if budget.uma and entry.mtp and not spilled and (measured := _MEASURED_DECODE_TOK_S.get(key)) is not None:
        return measured
    bandwidth = (_HOST_BANDWIDTH_GB_S if spilled
                 else _UMA_BANDWIDTH_GB_S if budget.uma
                 else _DISCRETE_BANDWIDTH_GB_S)
    bytes_per_token = max(1.0, variant.size_bytes * entry.decode_fraction)
    return bandwidth * 1e9 / bytes_per_token


def recommended_entry(budget: HardwareBudget,
                      entries: tuple[CatalogEntry, ...] | None = None, *, backend: str = "auto"
                      ) -> tuple[CatalogEntry, str] | None:
    """The catalog's default pick for THIS machine, with its reason key.

    Callers pass pre-filtered entries when some are ineligible for reasons the catalog can't know
    (engine too old). Entries marked ``recommend: false`` are never picked. Reasons:
    product-default (the product's maker chose this entry and it runs resident);
    best-quality-resident (quality won among resident entries clearing the pleasant floor);
    speed-gated-quality (same, but the floor eliminated a HIGHER quality candidate).
    Returns None when no eligible entry runs resident at the pleasant floor; slower and spilled
    models remain available for explicit selection.
    """
    pool = tuple(e for e in (CATALOG if entries is None else entries) if e.recommend)
    fitting = [(e, c) for e in pool if (c := select_variant(e, budget)) is not None]
    if not fitting:
        return None

    def speed(t, spilled=False):
        return predicted_decode_tok_s(t[0], t[1].variant, budget, spilled=spilled, backend=backend)

    resident = [(e, c) for e, c in fitting if c.zero_spill]
    product_default = _PRODUCT_DEFAULTS.get(_hardware_key(budget, backend))
    for entry, _ in resident:
        if entry.id == product_default:
            return (entry, "product-default")
    pleasant = [t for t in resident if speed(t) >= PLEASANT_FLOOR_TOK_S]
    if pleasant:
        pick = max(pleasant, key=lambda t: (t[0].quality, -t[1].variant.size_bytes))[0]
        floor_gated = any(e.quality > pick.quality for e, _ in resident)
        return (pick, "speed-gated-quality" if floor_gated else "best-quality-resident")
    # A slow or spilled model may be usable, but it is not a recommendation. Keep it
    # discoverable through Browse so the user can opt in with the degradation visible.
    return None


# ── catalog data: packaged JSON, refreshed from GitHub in memory ─
#
# catalog.json ships as package data and is loaded at import (no network on the import path). A
# TTL-gated background refresh fetches the same file from the repo's main branch and swaps it in
# MEMORY only: nothing on disk changes, so a git checkout never sees a dirty tracked file and the
# packaged copy remains the offline truth. A reverted commit on main heals every install on its
# next fetch, and day-0 entries reach users without an app release.

_CATALOG_URL = ("https://raw.githubusercontent.com/NousResearch/hermes-agent"
                "/main/hermes_cli/local_runtime/catalog.json")
_SCHEMA_VERSION = 3  # bump when an older app would read a newer catalog but launch its entries wrongly
_REFRESH_TTL_S = 6 * 3600
_refresh_lock = threading.Lock()
# None = never attempted: monotonic() counts from boot, so a 0.0 sentinel read as "refreshed at
# boot" and suppressed the first fetch for the first six awake hours after every reboot.
_last_refresh_attempt: float | None = None


def _asset_from(d: dict | None) -> AssetFile | None:
    if not d:
        return None
    return AssetFile(path=d["path"], size_bytes=int(d["size_bytes"]), local=d.get("local"),
                     repo=d.get("repo"), revision=d.get("revision", "main"))


# Scalar CatalogEntry fields parsed from JSON: key -> (coerce, default); None default = required.
_SCALAR_FIELDS = {
    "n_ctx_train": (int, None), "full_layers": (int, None),
    "recurrent_layers": (int, None), "per_layer_f16": (int, None),
    "swa_layers": (int, 0), "swa_window": (int, 0),
    "moe": (bool, False), "mtp": (bool, False), "mtp_draft_depth": (int, 3),
    "n_vocab": (int, 0), "sampling": (dict, {}), "min_engine": (str, ""),
    "quality": (int, 0), "decode_fraction": (float, 1.0), "architecture": (str, ""),
    "recommend": (bool, True), "draft_spec_type": (str, "draft-dspark"), "draft_depth": (int, 3),
    "draft_min_engine": (str, ""),
}


def _load_catalog(doc: dict) -> tuple[CatalogEntry, ...]:
    """Parse a catalog document. Unknown fields are ignored (newer catalogs stay readable by older
    apps); a major schema bump is the signal that they wouldn't be, and the caller skips it."""
    if int(doc.get("schema_version", 0)) != _SCHEMA_VERSION:
        raise ValueError(f"catalog schema {doc.get('schema_version')!r} "
                         f"(this build reads {_SCHEMA_VERSION})")
    entries = []
    for m in doc["models"]:
        variants = tuple(QuantVariant(quant=v["quant"], validated=bool(v.get("validated")),
                                      files=tuple(_asset_from(f) for f in v["files"]),
                                      lazy_bytes=int(v.get("lazy_bytes", 0)))
                         for v in m["variants"])
        scalars = {k: coerce(m[k] if default is None else m.get(k, default))
                   for k, (coerce, default) in _SCALAR_FIELDS.items()}
        entries.append(CatalogEntry(
            id=m["id"], display_name=m["display_name"],
            description=m["description"], repo=m["repo"], variants=variants,
            mmproj=_asset_from(m.get("mmproj")), draft=_asset_from(m.get("draft")),
            mtp_head=_asset_from(m.get("mtp_head")),
            **scalars))
    return tuple(entries)


def _packaged_catalog() -> tuple[CatalogEntry, ...]:
    from importlib.resources import files

    raw = files("hermes_cli.local_runtime").joinpath("catalog.json").read_text(encoding="utf-8-sig")
    return _load_catalog(json.loads(raw))


CATALOG: tuple[CatalogEntry, ...] = _packaged_catalog()


def refresh_catalog(force: bool = False) -> bool:
    """Fetch the current catalog from the repo and swap it in memory. Best-effort: any failure
    (offline, GitHub down, unreadable schema) leaves the running catalog untouched and retries
    after the TTL. Returns True when a fetched document replaced the catalog."""
    global CATALOG, _last_refresh_attempt

    now = time.monotonic()
    with _refresh_lock:
        if not force and _last_refresh_attempt is not None and now - _last_refresh_attempt < _REFRESH_TTL_S:
            return False
        _last_refresh_attempt = now
    try:
        req = urllib.request.Request(_CATALOG_URL, headers={"User-Agent": "hermes-local-runtime"})
        with urllib.request.urlopen(req, timeout=10) as r:
            fetched = _load_catalog(json.load(r))
    except Exception as exc:
        logger.debug("catalog refresh skipped: %s", exc)
        return False
    if fetched != CATALOG:
        logger.info("catalog refreshed from repo (%d models)", len(fetched))
    CATALOG = fetched
    return True


def refresh_catalog_soon() -> None:
    """TTL-gated background refresh; returns immediately. The current request serves the catalog
    it already has — the refresh lands for the next one."""
    last = _last_refresh_attempt
    if last is not None and time.monotonic() - last < _REFRESH_TTL_S:
        return
    threading.Thread(target=refresh_catalog, daemon=True, name="catalog-refresh").start()


def catalog_by_id() -> dict[str, CatalogEntry]:
    return {entry.id: entry for entry in CATALOG}


def find_entry_for_model(model_id: str) -> tuple[CatalogEntry, QuantVariant] | None:
    """Locate the entry + variant that owns a staged model id."""
    for entry in CATALOG:
        for variant in entry.variants:
            if variant.model_id == model_id:
                return entry, variant
    return None


def entry_for_model(model_id: str) -> CatalogEntry | None:
    hit = find_entry_for_model(model_id)
    return hit[0] if hit is not None else None