Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Send llama-server's API key when probing its local API

Senthanbb0eee3c5745
diff --git a/README.md b/README.md
index 1ecf9f9..74a63aa 100644
--- a/README.md
+++ b/README.md
@@ -705,6 +705,12 @@ The collector probes existing local listening ports with GET requests to `/props
absent, disabled, or authenticated; other evidence is still useful. It does not launch `ollama`,
`lms`, or an inference request. Throughput may update only after a request completes.
+If llama-server was started with `--api-key` or `--api-key-file`, the probe sends that key as a
+bearer token to the same server's own port. The key is never stored, printed, or written to JSON,
+and the inspector shows it as `***`. A key set only through `LLAMA_API_KEY` can't be seen from
+outside the process. In that case the probe stops at the first 401, so the server logs one
+rejected request every 30 seconds, and the inspector says why server details are missing.
+
Ollama's local `server.log` is considered when one Ollama runner is detected. Log formats,
rotation, and stale load blocks can affect attribution, so compare the reported model and
process when interpreting logs. Log sizes describe allocations at load time, not necessarily
diff --git a/src/gputaskman/app.py b/src/gputaskman/app.py
index 8127771..8e08827 100644
--- a/src/gputaskman/app.py
+++ b/src/gputaskman/app.py
@@ -18,7 +18,7 @@ from textual.widgets import DataTable, Footer, Markdown, Select, Static
from textual.worker import get_current_worker
from .collector import BACKEND, Collector, ProcRow, Snapshot, enumerate_adapters, ordered_adapters
-from .llm import gb
+from .llm import gb, redact_cmdline
from .nvml import NvDevice, format_metric
from .recovery import LiveCollectors
from .report import (DRM_RESIDENT_NOTE, METAL_NOTE, WSL_EMPTY, drm_note, has_engine_counters, has_spill_counters, has_wddm_counters,
@@ -755,7 +755,7 @@ class GpuTaskManager(App):
if r.detail.ports:
out.append(Text("listening: " + ", ".join(map(str, r.detail.ports[:8])), style="grey50"))
if r.detail.cmdline:
- out.append(Text(" ".join(r.detail.cmdline)[:400], style="grey42", overflow="ellipsis"))
+ out.append(Text(" ".join(redact_cmdline(r.detail.cmdline))[:400], style="grey42", overflow="ellipsis"))
elif r.detail.access_denied:
out.append(Text("(protected process - run elevated for runtime/file details)", style="grey42"))
panel.update(Group(*out))
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index ea15405..528f656 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -14,7 +14,7 @@ from .amdsmi import AmdSmi, _bdf
from .adl import Adl
from .dxgi import VENDOR_NVIDIA, Adapter
from .handy import HandyEvidence, inspect_handy
-from .llm import LlmView, ServerProber, build_view, gb, ollama_log_path
+from .llm import LlmView, ServerProber, api_key, build_view, gb, ollama_log_path
from . import nvml as nvml_mod
from .nvml import Nvml, NvDevice
from .pdh import ProcGpu
@@ -332,7 +332,9 @@ class Collector:
if (row.detail.is_diffusion or row.kind not in AI_KINDS or not (row.detail.models or row.kind == "LLM")
or not {"dedicated", "shared"} <= row.gpu.available):
continue
- own = self.prober.get(self.inspector.exclusive_ports(row.pid))
+ ports = self.inspector.exclusive_ports(row.pid)
+ key = api_key(row.detail.cmdline)
+ own = self.prober.get(ports, key)
host = self.prober.get(self._parent_ports(row.pid)) # Ollama / LM Studio front-end process
log = ollama_log_path() if row in ollama_runners and len(ollama_runners) == 1 else None
# On an APU, GTT is the GPU's ordinary memory (often most of it), not spill: count it as held memory.
@@ -340,6 +342,12 @@ class Collector:
row.llm = build_view(row.detail, held, row.gpu.shared if snap.shared_is_spill else 0,
own or host, log, host=host if own else None,
attributed=row.gpu.shown_vram is not None)
+ locked = None if own else self.prober.locked_port(ports)
+ if locked:
+ why = ("rejected the key from the launch arguments" if key else
+ "wants an API key, and none was found in the launch arguments (a key set through "
+ "LLAMA_API_KEY can't be read from outside the process)")
+ row.llm.notes.append(f"Local API on port {locked} {why}. Server details are skipped.")
self.inspector.prune(set(procs))
return snap
diff --git a/src/gputaskman/llm.py b/src/gputaskman/llm.py
index 57430c6..4fb4526 100644
--- a/src/gputaskman/llm.py
+++ b/src/gputaskman/llm.py
@@ -16,6 +16,7 @@ import os
import re
import sys
import time
+import urllib.error
import urllib.request
from dataclasses import dataclass, field
@@ -119,12 +120,68 @@ def ollama_log_path() -> str:
_opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
-def _get(url: str, timeout: float = 0.5):
+_SECRET_FLAGS = ("--api-key", "--api-key-file")
+UNAUTHORIZED = object() # _get result for HTTP 401: the server wants a key
+
+
+def api_key(cmd: list[str]) -> str | None:
+ """llama-server's API key from its own launch arguments, if it was given one there.
+
+ `--api-key` may list several comma-separated keys; any of them works. `--api-key-file` holds one key per
+ line. A key set only through the LLAMA_API_KEY environment variable can't be seen from outside the process.
+ The key is only sent back to that process's own localhost port and is never stored or shown.
+ """
+ for i, arg in enumerate(cmd):
+ key, val = arg, None
+ if "=" in arg and arg.startswith("--"):
+ key, val = arg.split("=", 1)
+ if key not in _SECRET_FLAGS:
+ continue
+ if val is None and i + 1 < len(cmd):
+ val = cmd[i + 1]
+ if not val:
+ continue
+ if key == "--api-key":
+ return next((k.strip() for k in val.split(",") if k.strip()), None)
+ try:
+ with open(val, encoding="utf-8", errors="replace") as f:
+ lines = f.read(64 << 10).splitlines()
+ except OSError:
+ continue
+ found = next((ln.strip() for ln in lines if ln.strip()), None)
+ if found:
+ return found
+ return None
+
+
+def redact_cmdline(cmd: list[str]) -> list[str]:
+ """Command line with API keys and key-file paths replaced, for display."""
+ out, hide = [], False
+ for arg in cmd:
+ if hide:
+ out.append("***")
+ hide = False
+ elif arg in _SECRET_FLAGS:
+ out.append(arg)
+ hide = True
+ elif "=" in arg and arg.split("=", 1)[0] in _SECRET_FLAGS:
+ out.append(arg.split("=", 1)[0] + "=***")
+ else:
+ out.append(arg)
+ return out
+
+
+def _get(url: str, timeout: float = 0.5, key: str | None = None):
+ headers = {"Accept": "application/json"}
+ if key:
+ headers["Authorization"] = f"Bearer {key}"
try:
- with _opener.open(urllib.request.Request(url, headers={"Accept": "application/json"}), timeout=timeout) as r:
+ with _opener.open(urllib.request.Request(url, headers=headers), timeout=timeout) as r:
if r.status != 200:
return None
body = r.read(4 << 20)
+ except urllib.error.HTTPError as e:
+ return UNAUTHORIZED if e.code == 401 else None
except Exception:
return None
try:
@@ -195,10 +252,16 @@ def lmstudio_models(v1: dict | None, v0: dict | None) -> list[dict] | None:
return None
-def probe_server(port: int, prev: ServerInfo | None = None) -> ServerInfo | None:
- """Read-only GETs. Never calls inference, load or download endpoints."""
+def probe_server(port: int, prev: ServerInfo | None = None, key: str | None = None) -> ServerInfo | None:
+ """Read-only GETs. Never calls inference, load or download endpoints.
+
+ Returns UNAUTHORIZED when the port answers 401 and no key (or a wrong one) was sent. Probing stops there,
+ so a locked server logs one rejected request per probe instead of one per endpoint.
+ """
base = f"http://127.0.0.1:{port}"
- props = _get(base + "/props")
+ props = _get(base + "/props", key=key)
+ if props is UNAUTHORIZED:
+ return UNAUTHORIZED
if isinstance(props, dict) and ("default_generation_settings" in props or "model_path" in props):
s = ServerInfo("llama-server", base, model_path=props.get("model_path"))
dgs = props.get("default_generation_settings") or {}
@@ -207,13 +270,13 @@ def probe_server(port: int, prev: ServerInfo | None = None) -> ServerInfo | None
if prev and prev.kind == "llama-server" and prev.model_path == s.model_path:
s.models = prev.models # identity only changes when the model does
else:
- v1 = _get(base + "/v1/models")
+ v1 = _get(base + "/v1/models", key=key)
s.models = llama_server_models(v1, s.model_path) if isinstance(v1, dict) else []
- slots = _get(base + "/slots")
+ slots = _get(base + "/slots", key=key)
if isinstance(slots, list):
s.slots_total = len(slots)
s.slots_busy = sum(1 for x in slots if x.get("is_processing"))
- metrics = _get(base + "/metrics")
+ metrics = _get(base + "/metrics", key=key)
if isinstance(metrics, str):
s.gen_tok_s = _metric(metrics, "predicted_tokens_seconds")
s.prompt_tok_s = _metric(metrics, "prompt_tokens_seconds")
@@ -238,15 +301,21 @@ class ServerProber:
def __init__(self) -> None:
self._cache: dict[int, tuple[float, ServerInfo | None]] = {}
self._last: dict[int, ServerInfo] = {}
+ self.locked: set[int] = set() # ports whose last probe was refused with 401
- def get(self, ports: list[int]) -> ServerInfo | None:
+ def get(self, ports: list[int], key: str | None = None) -> ServerInfo | None:
now = time.time()
for port in ports:
hit = self._cache.get(port)
if hit and now - hit[0] < (self.TTL_HIT if hit[1] else self.TTL_MISS):
info = hit[1]
else:
- info = probe_server(port, self._last.get(port))
+ info = probe_server(port, self._last.get(port), key)
+ if info is UNAUTHORIZED:
+ self.locked.add(port)
+ info = None
+ else:
+ self.locked.discard(port)
self._cache[port] = (now, info)
if info:
self._last[port] = info
@@ -254,6 +323,10 @@ class ServerProber:
return info
return None
+ def locked_port(self, ports: list[int]) -> int | None:
+ """First of these ports whose last probe was refused for a missing or wrong API key."""
+ return next((p for p in ports if p in self.locked), None)
+
# ---------------------------------------------------------------- the breakdown
diff --git a/tests/test_core.py b/tests/test_core.py
index 4a1e709..ed024eb 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import contextlib
+import dataclasses
import asyncio
import ctypes
import io
@@ -27,8 +28,9 @@ from gputaskman.collector import Collector, ProcRow, Snapshot, select_adapter, s
from gputaskman.dxgi import Adapter
from gputaskman.gguf import GgufInfo, read_gguf
from gputaskman.handy import HandyEvidence, inspect_handy, _log_cache, _settings_cache
-from gputaskman.llm import (LlmView, ServerInfo, build_view, is_gpu_device, llama_server_models, lmstudio_models,
- parse_flags, parse_load_log, probe_server)
+from gputaskman.llm import (LlmView, ServerInfo, ServerProber, api_key, build_view, is_gpu_device,
+ llama_server_models, lmstudio_models, parse_flags, parse_load_log, probe_server,
+ redact_cmdline)
from gputaskman.nvml import NvDevice, Nvml
from gputaskman.pdh import AdapterGpu, GpuCounters, ProcGpu
from gputaskman.procinfo import (AI_KINDS, MappedModel, ProcDetail, ProcInspector, _is_model_file, _match_runtimes,
@@ -746,10 +748,68 @@ class IdentityTests(unittest.TestCase):
"meta": {"ftype": "Q8_0", "n_ctx": 8192, "size": 447}}]}
self.assertEqual(llama_server_models(llama, r"C:\m\Laya-Q8_0.gguf")[0]["type"], "decision")
+ def test_api_key_from_launch_arguments(self):
+ self.assertEqual(api_key(["llama-server", "--api-key", "abc"]), "abc")
+ self.assertEqual(api_key(["llama-server", "--api-key=abc,def"]), "abc")
+ self.assertIsNone(api_key(["llama-server", "-m", "x.gguf"]))
+ self.assertIsNone(api_key(["llama-server", "--api-key"]))
+ with tempfile.TemporaryDirectory() as root:
+ path = os.path.join(root, "keys.txt")
+ Path(path).write_text("\n file-key \nsecond\n", encoding="utf-8")
+ self.assertEqual(api_key(["llama-server", "--api-key-file", path]), "file-key")
+ self.assertIsNone(api_key(["llama-server", "--api-key-file", os.path.join(root, "missing")]))
+ self.assertEqual(redact_cmdline(["llama-server", "--api-key", "abc", "--api-key-file=/k", "--port", "8080"]),
+ ["llama-server", "--api-key", "***", "--api-key-file=***", "--port", "8080"])
+ self.assertNotIn("api_key", str(parse_flags(["--api-key", "abc"])))
+
+ def test_llama_server_with_api_key(self):
+ """A real local HTTP server that only answers when the bearer token matches, like llama-server --api-key."""
+ import http.server
+ seen: list[tuple[str, str | None]] = []
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ auth = self.headers.get("Authorization")
+ seen.append((self.path, auth))
+ if auth != "Bearer s3cret":
+ self.send_response(401)
+ self.end_headers()
+ return
+ body = {"/props": {"model_path": "/m/x.gguf", "total_slots": 2,
+ "default_generation_settings": {"n_ctx": 4096}},
+ "/slots": [{"is_processing": True}, {"is_processing": False}]}.get(self.path)
+ self.send_response(200 if body is not None else 404)
+ self.end_headers()
+ if body is not None:
+ self.wfile.write(json.dumps(body).encode())
+
+ def log_message(self, *args):
+ pass
+
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ port = server.server_address[1]
+ try:
+ prober = ServerProber()
+ self.assertIsNone(prober.get([port]))
+ self.assertEqual(prober.locked, {port})
+ self.assertEqual([p for p, _ in seen], ["/props"]) # stops at the first 401, no LM Studio/Ollama probes
+ seen.clear()
+ s = probe_server(port, key="s3cret")
+ self.assertEqual((s.kind, s.n_ctx, s.slots_busy, s.slots_total), ("llama-server", 4096, 1, 2))
+ self.assertTrue(all(auth == "Bearer s3cret" for _, auth in seen))
+ self.assertNotIn("s3cret", json.dumps(dataclasses.asdict(s)))
+ prober._cache.clear()
+ self.assertIsNotNone(prober.get([port], "s3cret"))
+ self.assertEqual(prober.locked, set())
+ finally:
+ server.shutdown()
+ server.server_close()
+
def test_lmstudio_falls_back_to_v0_when_v1_is_an_error(self):
replies = {"/api/v1/models": {"error": "Unexpected endpoint or method. (GET /api/v1/models)"},
"/api/v0/models": {"data": [{"id": "qwen", "state": "loaded", "max_context_length": 8192}]}}
- with patch("gputaskman.llm._get", side_effect=lambda url: next(
+ with patch("gputaskman.llm._get", side_effect=lambda url, **_: next(
(v for k, v in replies.items() if url.endswith(k)), None)):
s = probe_server(1234)
self.assertEqual((s.kind, s.models[0]["name"]), ("lmstudio", "qwen"))