Skip to content

Personal AI, On Personal Devices

Python
25.4 KB
Raw Blame
"""``jarvis model`` — model management subcommands."""

from __future__ import annotations

import os
import re
import shlex
import shutil
import subprocess
import sys
import tempfile
import uuid
from pathlib import Path

import click
import httpx
from rich.console import Console
from rich.panel import Panel
from rich.table import Table

from openjarvis.core.config import DEFAULT_CONFIG_DIR, load_config
from openjarvis.core.registry import ModelRegistry
from openjarvis.engine import discover_engines, discover_models
from openjarvis.intelligence import merge_discovered_models, register_builtin_models
from openjarvis.intelligence.model_catalog import BUILTIN_MODELS


@click.group()
def model() -> None:
    """Manage language models."""


@model.command("list")
def list_models() -> None:
    """List available models from running engines."""
    console = Console()
    config = load_config()
    register_builtin_models()

    engines = discover_engines(config)
    if not engines:
        console.print(
            "[yellow]No inference engines detected.[/yellow]\n"
            "Start an engine (e.g. [cyan]ollama serve[/cyan]) and try again."
        )
        return

    all_models = discover_models(engines)
    for ek, model_ids in all_models.items():
        merge_discovered_models(ek, model_ids)

    table = Table(title="Available Models")
    table.add_column("Engine", style="cyan")
    table.add_column("Model", style="green")
    table.add_column("Params", justify="right")
    table.add_column("Active", justify="right")
    table.add_column("Context", justify="right")
    table.add_column("VRAM", justify="right")
    table.add_column("Arch", style="dim")

    for engine_key, model_ids in all_models.items():
        for mid in model_ids:
            try:
                spec = ModelRegistry.get(mid)
                params = f"{spec.parameter_count_b}B" if spec.parameter_count_b else "-"
                active = (
                    f"{spec.active_parameter_count_b}B"
                    if spec.active_parameter_count_b
                    else "-"
                )
                ctx = f"{spec.context_length:,}" if spec.context_length else "-"
                vram = f"{spec.min_vram_gb}GB" if spec.min_vram_gb else "-"
                arch = spec.metadata.get("architecture", "-")
            except KeyError:
                params = "-"
                active = "-"
                ctx = "-"
                vram = "-"
                arch = "-"
            table.add_row(engine_key, mid, params, active, ctx, vram, arch)

    console.print(table)


@model.command()
@click.argument("model_name")
def info(model_name: str) -> None:
    """Show details for a model."""
    console = Console()
    register_builtin_models()

    # Also try discovering from running engines
    config = load_config()
    engines = discover_engines(config)
    all_models = discover_models(engines)
    for ek, model_ids in all_models.items():
        merge_discovered_models(ek, model_ids)

    if not ModelRegistry.contains(model_name):
        console.print(f"[red]Model not found:[/red] {model_name}")
        sys.exit(1)

    spec = ModelRegistry.get(model_name)
    params = f"{spec.parameter_count_b}B" if spec.parameter_count_b else "unknown"
    active = (
        f"{spec.active_parameter_count_b}B" if spec.active_parameter_count_b else "-"
    )
    ctx_len = f"{spec.context_length:,}" if spec.context_length else "unknown"
    vram = f"{spec.min_vram_gb}GB" if spec.min_vram_gb else "-"
    engines_str = ", ".join(spec.supported_engines) if spec.supported_engines else "-"
    provider = spec.provider or "-"
    api_key = "required" if spec.requires_api_key else "not required"
    lines = [
        f"[bold]Model ID:[/bold]       {spec.model_id}",
        f"[bold]Name:[/bold]           {spec.name}",
        f"[bold]Parameters:[/bold]     {params}",
        f"[bold]Active Params:[/bold]  {active}",
        f"[bold]Context:[/bold]        {ctx_len}",
        f"[bold]Quantization:[/bold]   {spec.quantization.value}",
        f"[bold]Min VRAM:[/bold]       {vram}",
        f"[bold]Engines:[/bold]        {engines_str}",
        f"[bold]Provider:[/bold]       {provider}",
        f"[bold]API Key:[/bold]        {api_key}",
    ]

    # Append metadata fields with well-known labels
    meta_labels = {
        "architecture": "Architecture",
        "hf_repo": "HuggingFace",
        "url": "More Info",
        "teacher": "Teacher Model",
        "quantization": "Quant Format",
        "license": "License",
        "pricing_input": "Price (input)",
        "pricing_output": "Price (output)",
    }
    for key, label in meta_labels.items():
        value = spec.metadata.get(key)
        if value is not None:
            if key.startswith("pricing_"):
                value = f"${value}/M tokens"
            elif key == "hf_repo":
                value = f"https://huggingface.co/{value}"
            pad = " " * max(1, 14 - len(label))
            lines.append(f"[bold]{label}:[/bold]{pad}{value}")

    # Any remaining metadata not covered above
    extra_keys = set(spec.metadata) - set(meta_labels)
    for key in sorted(extra_keys):
        pad = " " * max(1, 14 - len(key))
        lines.append(f"[bold]{key}:[/bold]{pad}{spec.metadata[key]}")

    console.print(Panel("\n".join(lines), title=spec.name, border_style="blue"))


def ollama_pull(host: str, model_name: str, console: Console) -> bool:
    """Pull a model via Ollama API. Returns True on success."""
    console.print(f"Pulling [cyan]{model_name}[/cyan] via Ollama...")
    try:
        with httpx.stream(
            "POST",
            f"{host}/api/pull",
            json={"name": model_name, "stream": True},
            timeout=600.0,
        ) as resp:
            resp.raise_for_status()
            import json

            for line in resp.iter_lines():
                if not line.strip():
                    continue
                try:
                    data = json.loads(line)
                except Exception:
                    continue
                if "error" in data:
                    console.print(f"Ollama error: {data['error']}", markup=False)
                    return False
                status = data.get("status", "")
                if "total" in data and "completed" in data:
                    total = data["total"]
                    done = data["completed"]
                    pct = int(done / total * 100) if total else 0
                    console.print(f"  {status}: {pct}%", end="\r")
                elif status:
                    console.print(f"  {status}")
        console.print(f"\n[green]Successfully pulled {model_name}[/green]")
        return True
    except httpx.ConnectError:
        console.print("[red]Cannot connect to Ollama.[/red] Is it running?")
        return False
    except httpx.HTTPStatusError as exc:
        console.print(f"[red]Ollama error:[/red] {exc.response.status_code}")
        return False


def find_model_spec(model_name: str):
    """Look up a model in the builtin catalog. Returns None if not found."""
    for spec in BUILTIN_MODELS:
        if spec.model_id == model_name:
            return spec
    return None


def hf_download(repo: str, filename: str | None, console: Console) -> bool:
    """Download from HuggingFace via huggingface-cli. Returns True on success."""
    cmd = ["huggingface-cli", "download", repo]
    if filename:
        cmd.append(filename)
    try:
        subprocess.run(cmd, check=True)
        console.print("[green]Download complete.[/green]")
        return True
    except FileNotFoundError:
        console.print(
            "[red]huggingface-cli not found.[/red]\n"
            "Install it: [cyan]pip install huggingface_hub[/cyan]\n"
            f"Or download manually: https://huggingface.co/{repo}"
        )
        return False
    except subprocess.CalledProcessError:
        console.print("[red]Download failed.[/red]")
        return False


@model.command()
@click.argument("model_name")
@click.option("--engine", default=None, help="Engine to download for.")
def pull(model_name: str, engine: str | None) -> None:
    """Download a model."""
    console = Console()
    config = load_config()
    engine = engine or config.engine.default or "ollama"

    if engine == "ollama":
        host = (
            config.engine.ollama_host
            or os.environ.get("OLLAMA_HOST")
            or "http://localhost:11434"
        ).rstrip("/")
        if not ollama_pull(host, model_name, console):
            sys.exit(1)
    elif engine in ("llamacpp", "mlx"):
        spec = find_model_spec(model_name)
        if not spec:
            console.print(f"[red]Model not in catalog:[/red] {model_name}")
            sys.exit(1)
        if engine == "llamacpp":
            repo = spec.metadata.get("hf_repo", "")
            gguf = spec.metadata.get("gguf_file", "")
            if not repo or not gguf:
                console.print(f"[red]No GGUF download info for {model_name}[/red]")
                sys.exit(1)
            console.print(f"Downloading [cyan]{gguf}[/cyan] from {repo}...")
            if not hf_download(repo, gguf, console):
                sys.exit(1)
        else:  # mlx
            mlx_repo = spec.metadata.get("mlx_repo", "")
            if not mlx_repo:
                console.print(f"[red]No MLX repo info for {model_name}[/red]")
                sys.exit(1)
            console.print(f"Downloading [cyan]{mlx_repo}[/cyan]...")
            if not hf_download(mlx_repo, None, console):
                sys.exit(1)
    elif engine in ("vllm", "sglang"):
        console.print(
            f"[cyan]{model_name}[/cyan] will download automatically when "
            f"{engine} starts serving it."
        )
    else:
        console.print(
            f"Manual download required for engine [cyan]{engine}[/cyan].\n"
            f"Check the engine documentation for instructions."
        )


def _slug_from_repo(
    hf_repo: str, engine: str, quantize: str | None, mlx_4bit: bool
) -> str:
    """Generate output directory slug from HF repo and conversion params."""
    base = re.sub(r"[^A-Za-z0-9._-]+", "-", hf_repo.replace("/", "--"))
    base = base.strip("._-") or "model"
    if engine == "mlx":
        return f"{base}-mlx-4bit" if mlx_4bit else f"{base}-mlx"
    elif engine == "llamacpp":
        if quantize:
            return f"{base}-{quantize}"
        return f"{base}-gguf"
    return base


def _download_snapshot(hf_repo: str, console: Console) -> str | None:
    """Download *hf_repo* and return the concrete local snapshot directory."""
    try:
        from huggingface_hub import snapshot_download
    except ImportError:
        console.print(
            "[red]Model conversion needs huggingface_hub.[/red]\n"
            "Install it: [cyan]pip install huggingface_hub[/cyan]"
        )
        return None

    try:
        console.print(f"Downloading [cyan]{hf_repo}[/cyan]...")
        return str(snapshot_download(repo_id=hf_repo))
    except Exception as exc:
        console.print(f"[red]Hugging Face download failed:[/red] {exc}")
        return None


def _convert_mlx(hf_repo: str, output: str, mlx_4bit: bool, console: Console) -> bool:
    """Convert an HF model to MLX format. Returns True on success."""
    try:
        import mlx_lm
    except ImportError:
        console.print(
            "[red]MLX conversion needs the inference-mlx extra.[/red]\n"
            'Install it: [cyan]pip install "openjarvis[inference-mlx]"[/cyan]'
        )
        return False

    snapshot = _download_snapshot(hf_repo, console)
    if snapshot is None:
        return False

    try:
        # mlx_lm.convert deliberately refuses an existing output path. The
        # command gives it a fresh staging destination and only publishes that
        # directory after conversion succeeds.
        Path(output).parent.mkdir(parents=True, exist_ok=True)
        console.print(
            f"Converting [cyan]{hf_repo}[/cyan] to MLX at [cyan]{output}[/cyan]..."
        )
        mlx_lm.convert(hf_path=snapshot, mlx_path=output, quantize=mlx_4bit)
        if not Path(output).is_dir() or not any(Path(output).iterdir()):
            raise RuntimeError("converter completed without producing an artifact")
        console.print(f"[green]Converted → {output}[/green]")
        return True
    except Exception as exc:
        console.print(f"[red]MLX conversion failed:[/red] {exc}")
        return False


def _convert_gguf(
    hf_repo: str,
    output: str,
    quantize: str | None,
    console: Console,
    config: object | None = None,
) -> str | None:
    """Convert an HF model to GGUF and return the final artifact path."""
    snapshot = _download_snapshot(hf_repo, console)
    if snapshot is None:
        return None

    # Step 2: locate and run convert_hf_to_gguf.py
    convert_script = _locate_convert_script(console, config)
    if not convert_script:
        return None

    # Determine output file path
    os.makedirs(output, exist_ok=True)
    artifact_stem = _slug_from_repo(hf_repo, "", None, False)
    output_gguf = os.path.join(output, f"{artifact_stem}.f16.gguf")

    console.print(f"Converting to [cyan]{output_gguf}[/cyan]...")
    try:
        subprocess.run(
            [
                sys.executable,
                convert_script,
                snapshot,
                "--outtype",
                "f16",
                "--outfile",
                output_gguf,
            ],
            check=True,
        )
        if not os.path.isfile(output_gguf):
            raise RuntimeError("converter completed without producing a GGUF file")
    except FileNotFoundError:
        console.print(
            "[red]convert_hf_to_gguf.py not found.[/red]\n"
            "Install llama.cpp and set [cyan]$LLAMACPP_PATH[/cyan] to the repo "
            "root, or configure [cyan]engine.llamacpp.binary_path[/cyan]."
        )
        return None
    except (subprocess.CalledProcessError, RuntimeError) as exc:
        if isinstance(exc, RuntimeError):
            console.print(f"[red]Conversion to GGUF failed:[/red] {exc}")
            return None
        console.print("[red]Conversion to GGUF failed.[/red]")
        return None

    # Step 3: optional quantization
    if quantize:
        console.print(f"Quantizing to [cyan]{quantize}[/cyan]...")
        quant_script = _locate_quantize_script(console, config)
        if not quant_script:
            return None

        quantized_out = os.path.join(output, f"{artifact_stem}.{quantize}.gguf")
        try:
            subprocess.run(
                [quant_script, output_gguf, quantized_out, quantize],
                check=True,
            )
            if not os.path.isfile(quantized_out):
                raise RuntimeError("quantizer completed without producing a GGUF file")
            console.print(f"[green]Quantized → {quantized_out}[/green]")
        except FileNotFoundError:
            console.print(
                "[red]llama-quantize not found.[/red]\n"
                "Install llama.cpp and set [cyan]$LLAMACPP_PATH[/cyan] to the "
                "repo root, or configure "
                "[cyan]engine.llamacpp.binary_path[/cyan]."
            )
            return None
        except (subprocess.CalledProcessError, RuntimeError) as exc:
            if isinstance(exc, RuntimeError):
                console.print(f"[red]Quantization failed:[/red] {exc}")
                return None
            console.print("[red]Quantization failed.[/red]")
            return None
        return quantized_out
    else:
        console.print(f"[green]Converted → {output_gguf}[/green]")

    return output_gguf


def _llamacpp_search_roots(config: object | None) -> list[Path]:
    """Return deduplicated llama.cpp roots from env and engine config."""
    configured_path = ""
    try:
        configured_path = str(config.engine.llamacpp.binary_path)  # type: ignore[attr-defined]
    except AttributeError:
        pass

    raw_paths = [
        os.environ.get("LLAMACPP_PATH", ""),
        os.environ.get("LLAMA_CPP_DIR", ""),
        configured_path,
    ]
    roots: list[Path] = []
    for raw_path in raw_paths:
        if not raw_path:
            continue
        path = Path(raw_path).expanduser()
        # A configured path commonly points at build/bin/llama-server. Walk a
        # few ancestors so the repository-level conversion script is found.
        if path.name.startswith("llama-") or path.name.endswith(".py"):
            path = path.parent
        for root in (path, *list(path.parents)[:4]):
            if root not in roots:
                roots.append(root)
    return roots


def _locate_convert_script(
    console: Console, config: object | None = None
) -> str | None:
    """Locate convert_hf_to_gguf.py via env, config, or PATH."""
    for llama_dir in _llamacpp_search_roots(config):
        script = llama_dir / "convert_hf_to_gguf.py"
        if script.is_file():
            return str(script)
    # Try PATH
    script = shutil.which("convert_hf_to_gguf.py")
    if script:
        return script
    console.print(
        "[red]convert_hf_to_gguf.py not found.[/red]\n"
        "Install llama.cpp and set [cyan]$LLAMACPP_PATH[/cyan] to the repo "
        "root, or configure [cyan]engine.llamacpp.binary_path[/cyan]."
    )
    return None


def _locate_quantize_script(
    console: Console, config: object | None = None
) -> str | None:
    """Locate llama-quantize via env, config, or PATH."""
    for llama_dir in _llamacpp_search_roots(config):
        # llama-quantize is typically in the main llama.cpp build directory
        for name in (
            "llama-quantize",
            "bin/llama-quantize",
            "build/bin/llama-quantize",
        ):
            script = llama_dir / name
            if script.is_file():
                return str(script)
    # Try PATH
    script = shutil.which("llama-quantize")
    if script:
        return script
    console.print(
        "[red]llama-quantize not found.[/red]\n"
        "Install llama.cpp and set [cyan]$LLAMACPP_PATH[/cyan] to the repo "
        "root, or configure [cyan]engine.llamacpp.binary_path[/cyan]."
    )
    return None


def _remove_path(path: Path) -> None:
    """Remove one file, symlink, or directory without following symlinks."""
    if path.is_symlink() or path.is_file():
        path.unlink()
    elif path.is_dir():
        shutil.rmtree(path)


def _publish_output(staged: Path, output: Path) -> None:
    """Publish a completed staging directory, restoring the old output on error."""
    output.parent.mkdir(parents=True, exist_ok=True)
    backup: Path | None = None
    if os.path.lexists(output):
        backup = output.with_name(f".{output.name}.backup-{uuid.uuid4().hex}")
        os.replace(output, backup)
    try:
        os.replace(staged, output)
    except Exception:
        if backup is not None and os.path.lexists(backup):
            os.replace(backup, output)
        raise
    else:
        if backup is not None:
            _remove_path(backup)


def _import_ollama(
    gguf_path: str,
    staging_output: Path,
    model_name: str,
    console: Console,
) -> bool:
    """Import a staged GGUF into Ollama using a relocatable Modelfile."""
    artifact = Path(gguf_path)
    relative_artifact = artifact.relative_to(staging_output)
    modelfile = staging_output / "Modelfile"
    modelfile.write_text(f"FROM ./{relative_artifact.as_posix()}\n", encoding="utf-8")
    try:
        subprocess.run(
            ["ollama", "create", model_name, "-f", str(modelfile)],
            check=True,
        )
    except FileNotFoundError:
        console.print(
            "[red]ollama not found.[/red] Install Ollama and ensure the "
            "[cyan]ollama[/cyan] command is on PATH."
        )
        return False
    except subprocess.CalledProcessError:
        console.print("[red]Ollama model import failed.[/red]")
        return False
    return True


@model.command()
@click.argument("hf_repo")
@click.option("--engine", default=None, help="Engine to convert for.")
@click.option(
    "--quantize",
    default=None,
    help="GGUF quantization type (e.g., q4_k_m, q8_0).",
)
@click.option(
    "-q",
    "--mlx-4bit",
    is_flag=True,
    default=False,
    help="Enable 4-bit quantization for MLX.",
)
@click.option(
    "--output",
    default=None,
    help="Output directory for converted artifact.",
)
@click.option(
    "--force",
    is_flag=True,
    default=False,
    help="Replace existing output only after conversion succeeds.",
)
def convert(
    hf_repo: str,
    engine: str | None,
    quantize: str | None,
    mlx_4bit: bool,
    output: str | None,
    force: bool,
) -> None:
    """Convert a Hugging Face model to a local engine format."""
    console = Console()
    config = load_config()

    # Resolve engine and reject combinations that would otherwise be silently
    # ignored or produce an artifact the selected runtime cannot consume.
    engine = (engine or config.engine.default or "mlx").lower()
    if engine in ("vllm", "sglang"):
        console.print(
            f"[green]No conversion needed for {engine}.[/green] It consumes "
            f"Hugging Face repositories directly.\n"
            f"[cyan]Start with:[/cyan] jarvis host {shlex.quote(hf_repo)} "
            f"--backend {engine}"
        )
        return
    if engine not in ("mlx", "llamacpp", "ollama"):
        console.print(f"[red]Unsupported conversion engine:[/red] {engine}")
        raise SystemExit(1)
    if engine == "mlx" and quantize:
        console.print("[red]--quantize is only valid for GGUF conversion.[/red]")
        raise SystemExit(1)
    if engine != "mlx" and mlx_4bit:
        console.print("[red]--mlx-4bit is only valid with --engine mlx.[/red]")
        raise SystemExit(1)
    if quantize and not re.fullmatch(r"[A-Za-z0-9_]+", quantize):
        console.print("[red]Invalid GGUF quantization name.[/red]")
        raise SystemExit(1)
    quantize = quantize.lower() if quantize else None

    # Resolve output path
    if output is None:
        slug = _slug_from_repo(hf_repo, engine, quantize, mlx_4bit)
        output = str(DEFAULT_CONFIG_DIR / "models" / slug)

    # Keep the final path absolute without resolving a user-supplied symlink:
    # --force must replace the link itself, never its target.
    output_path = Path(os.path.abspath(Path(output).expanduser()))

    # Check existing output. --force does not remove it yet: conversion happens
    # in a sibling staging directory, so failures leave the old artifact intact.
    if os.path.lexists(output_path):
        # A symlink is always an existing output object, even when it points to
        # an empty directory. Never follow it to decide whether overwriting is
        # safe; --force replaces the link itself after conversion succeeds.
        if output_path.is_symlink() and not force:
            console.print(
                f"[red]Output path exists as a symlink:[/red] {output_path}\n"
                "Use [cyan]--force[/cyan] to overwrite."
            )
            sys.exit(1)
        if output_path.is_dir() and not output_path.is_symlink():
            contents = os.listdir(output_path)
            if contents and not force:
                console.print(
                    "[red]Output directory exists and is non-empty:[/red] "
                    f"{output_path}\n"
                    "Use [cyan]--force[/cyan] to overwrite."
                )
                sys.exit(1)
        elif not force:
            console.print(
                f"[red]Output path exists as a file:[/red] {output_path}\n"
                "Use [cyan]--force[/cyan] to overwrite."
            )
            sys.exit(1)

    output_path.parent.mkdir(parents=True, exist_ok=True)
    temp_root = Path(
        tempfile.mkdtemp(prefix=f".{output_path.name}.convert-", dir=output_path.parent)
    )
    staged_output = temp_root / "artifact"
    artifact_relative: Path | None = None
    ollama_name: str | None = None
    try:
        if engine == "mlx":
            if not _convert_mlx(hf_repo, str(staged_output), mlx_4bit, console):
                raise SystemExit(1)
        else:
            gguf_path = _convert_gguf(
                hf_repo,
                str(staged_output),
                quantize,
                console,
                config,
            )
            if gguf_path is None:
                raise SystemExit(1)
            artifact_relative = Path(gguf_path).relative_to(staged_output)
            if engine == "ollama":
                ollama_name = _slug_from_repo(
                    hf_repo.rsplit("/", 1)[-1], "", None, False
                ).lower()
                if not _import_ollama(
                    gguf_path,
                    staged_output,
                    ollama_name,
                    console,
                ):
                    raise SystemExit(1)

        _publish_output(staged_output, output_path)
    finally:
        if temp_root.exists():
            shutil.rmtree(temp_root)

    if engine == "mlx":
        artifact = output_path
        hint = f"jarvis host {shlex.quote(str(artifact))} --backend mlx"
        title = "MLX conversion complete"
    else:
        assert artifact_relative is not None
        artifact = output_path / artifact_relative
        if engine == "ollama":
            assert ollama_name is not None
            hint = f"jarvis chat --engine ollama --model {shlex.quote(ollama_name)}"
            title = "GGUF conversion and Ollama import complete"
        else:
            hint = f"jarvis host {shlex.quote(str(artifact))} --backend llamacpp"
            title = "GGUF conversion complete"
    console.print(
        Panel(
            f"[bold]Artifact:[/bold] {artifact}\n\n[cyan]Start with:[/cyan]\n  {hint}",
            title=title,
            border_style="green",
        )
    )