Skip to content

Personal AI, On Personal Devices

Python
5.1 KB
Raw Blame
"""Interactive runtime panel: GPU offload and context window (Ollama)."""

from __future__ import annotations

import os
from dataclasses import dataclass
from typing import Any, Optional

MAX_NUM_CTX = 200_000
MAX_NUM_GPU_LAYERS = 999


@dataclass(frozen=True, slots=True)
class ChatRuntimeOptions:
    """Per-session inference options chosen after model pick."""

    num_ctx: Optional[int] = None
    num_gpu: Optional[int] = None

    def to_engine_kwargs(self, *, engine_name: str) -> dict[str, Any]:
        """Return only options implemented by the selected engine."""
        if engine_name != "ollama":
            return {}
        out: dict[str, Any] = {}
        if self.num_ctx is not None and self.num_ctx > 0:
            out["num_ctx"] = min(int(self.num_ctx), MAX_NUM_CTX)
        if self.num_gpu is not None:
            # Ollama: large num_gpu = offload all layers; -1 means "all".
            if self.num_gpu < 0:
                out["num_gpu"] = MAX_NUM_GPU_LAYERS
            else:
                out["num_gpu"] = min(int(self.num_gpu), MAX_NUM_GPU_LAYERS)
        return out

    def summary(self, *, engine_name: str) -> str:
        if engine_name != "ollama":
            return "engine defaults (runtime controls require Ollama)"
        parts: list[str] = []
        if self.num_ctx is not None and self.num_ctx > 0:
            parts.append(f"ctx={self.num_ctx:,}")
        else:
            parts.append("ctx=default")
        if engine_name == "ollama":
            if self.num_gpu is None:
                parts.append("gpu=default")
            elif self.num_gpu < 0:
                parts.append("gpu=all layers")
            elif self.num_gpu == 0:
                parts.append("gpu=0 (CPU)")
            else:
                parts.append(f"gpu={self.num_gpu} layers")
        return ", ".join(parts)


def tty_wants_runtime_panel(cli_skip: bool = False) -> bool:
    """Show runtime panel on TTY unless skipped."""
    import sys

    if cli_skip:
        return False
    skip = (os.environ.get("JARVIS_SKIP_RUNTIME_PANEL", "") or "").strip().lower() in (
        "1",
        "true",
        "yes",
    )
    return sys.stdin.isatty() and not skip


def _parse_int(raw: str, *, default: Optional[int]) -> Optional[int]:
    text = (raw or "").strip()
    if not text:
        return default
    try:
        return int(text.replace("_", "").replace(",", ""))
    except ValueError:
        return default


def interactive_pick_runtime_options(
    console: Any,
    *,
    engine_name: str,
    cli_num_ctx: Optional[int] = None,
    cli_num_gpu: Optional[int] = None,
) -> ChatRuntimeOptions:
    """Prompt for context size and GPU offload after model selection."""
    # These option names and semantics are Ollama-specific. In particular,
    # forwarding them to OpenAI-compatible engines becomes an invalid API
    # request rather than a harmless no-op.
    if engine_name != "ollama":
        return ChatRuntimeOptions()
    if cli_num_ctx is not None or cli_num_gpu is not None:
        return ChatRuntimeOptions(num_ctx=cli_num_ctx, num_gpu=cli_num_gpu)

    console.print(
        "\n[bold]Runtime[/bold] "
        "[dim](Enter = engine default; applies this session)[/dim]",
    )
    console.print(
        "  [dim]num_ctx[/dim] — context window (tokens, max "
        f"{MAX_NUM_CTX:,}). [dim]num_gpu[/dim] — layers on GPU "
        "(-1 = all, 0 = CPU only).",
    )

    try:
        raw_ctx = input(
            f"Max context tokens [0–{MAX_NUM_CTX}, 0=default]: ",
        )
    except (EOFError, KeyboardInterrupt):
        return ChatRuntimeOptions()

    num_ctx = _parse_int(raw_ctx, default=None)
    if num_ctx is not None:
        if num_ctx <= 0:
            num_ctx = None
        else:
            num_ctx = min(num_ctx, MAX_NUM_CTX)

    num_gpu: Optional[int] = None
    try:
        raw_gpu = input(
            "GPU layer offload [-1=all, 0=CPU, N=layers, Enter=default]: ",
        )
    except (EOFError, KeyboardInterrupt):
        return ChatRuntimeOptions(num_ctx=num_ctx)
    parsed = _parse_int(raw_gpu, default=None)
    if parsed is not None:
        num_gpu = parsed

    opts = ChatRuntimeOptions(num_ctx=num_ctx, num_gpu=num_gpu)
    console.print(f"[dim]Runtime: {opts.summary(engine_name=engine_name)}[/dim]\n")
    return opts


def runtime_cli_options(command: Any) -> Any:
    """Click decorator: ``--num-ctx``, ``--num-gpu``, ``--skip-runtime-panel``."""
    import click

    command = click.option(
        "--num-ctx",
        type=int,
        default=None,
        help="Context window (tokens, max 200k). Skips runtime panel when set.",
    )(command)
    command = click.option(
        "--num-gpu",
        type=int,
        default=None,
        help="Ollama GPU layers (-1=all, 0=CPU). Skips runtime panel when set.",
    )(command)
    command = click.option(
        "--skip-runtime-panel",
        is_flag=True,
        default=False,
        help="Do not prompt for context/GPU after model pick.",
    )(command)
    return command


__all__ = [
    "MAX_NUM_CTX",
    "MAX_NUM_GPU_LAYERS",
    "ChatRuntimeOptions",
    "interactive_pick_runtime_options",
    "runtime_cli_options",
    "tty_wants_runtime_panel",
]