A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Send llama-server's API key when probing its local API
Senthanbb0eee3c5745
diff --git a/README.md b/README.mdindex 1ecf9f9..74a63aa 100644--- a/README.md+++ b/README.md@@ -705,6 +705,12 @@ The collector probes existing local listening ports with GET requests to `/propsabsent, disabled, or authenticated; other evidence is still useful. It does not launch `ollama`,`lms`, or an inference request. Throughput may update only after a request completes.+If llama-server was started with `--api-key` or `--api-key-file`, the probe sends that key as a+bearer token to the same server's own port. The key is never stored, printed, or written to JSON,+and the inspector shows it as `***`. A key set only through `LLAMA_API_KEY` can't be seen from+outside the process. In that case the probe stops at the first 401, so the server logs one+rejected request every 30 seconds, and the inspector says why server details are missing.+Ollama's local `server.log` is considered when one Ollama runner is detected. Log formats,rotation, and stale load blocks can affect attribution, so compare the reported model andprocess when interpreting logs. Log sizes describe allocations at load time, not necessarilydiff --git a/src/gputaskman/app.py b/src/gputaskman/app.pyindex 8127771..8e08827 100644--- a/src/gputaskman/app.py+++ b/src/gputaskman/app.py@@ -18,7 +18,7 @@ from textual.widgets import DataTable, Footer, Markdown, Select, Staticfrom textual.worker import get_current_workerfrom .collector import BACKEND, Collector, ProcRow, Snapshot, enumerate_adapters, ordered_adapters-from .llm import gb+from .llm import gb, redact_cmdlinefrom .nvml import NvDevice, format_metricfrom .recovery import LiveCollectorsfrom .report import (DRM_RESIDENT_NOTE, METAL_NOTE, WSL_EMPTY, drm_note, has_engine_counters, has_spill_counters, has_wddm_counters,@@ -755,7 +755,7 @@ class GpuTaskManager(App):if r.detail.ports:out.append(Text("listening: " + ", ".join(map(str, r.detail.ports[:8])), style="grey50"))if r.detail.cmdline:- out.append(Text(" ".join(r.detail.cmdline)[:400], style="grey42", overflow="ellipsis"))+ out.append(Text(" ".join(redact_cmdline(r.detail.cmdline))[:400], style="grey42", overflow="ellipsis"))elif r.detail.access_denied:out.append(Text("(protected process - run elevated for runtime/file details)", style="grey42"))panel.update(Group(*out))diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex ea15405..528f656 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -14,7 +14,7 @@ from .amdsmi import AmdSmi, _bdffrom .adl import Adlfrom .dxgi import VENDOR_NVIDIA, Adapterfrom .handy import HandyEvidence, inspect_handy-from .llm import LlmView, ServerProber, build_view, gb, ollama_log_path+from .llm import LlmView, ServerProber, api_key, build_view, gb, ollama_log_pathfrom . import nvml as nvml_modfrom .nvml import Nvml, NvDevicefrom .pdh import ProcGpu@@ -332,7 +332,9 @@ class Collector:if (row.detail.is_diffusion or row.kind not in AI_KINDS or not (row.detail.models or row.kind == "LLM")or not {"dedicated", "shared"} <= row.gpu.available):continue- own = self.prober.get(self.inspector.exclusive_ports(row.pid))+ ports = self.inspector.exclusive_ports(row.pid)+ key = api_key(row.detail.cmdline)+ own = self.prober.get(ports, key)host = self.prober.get(self._parent_ports(row.pid)) # Ollama / LM Studio front-end processlog = ollama_log_path() if row in ollama_runners and len(ollama_runners) == 1 else None# On an APU, GTT is the GPU's ordinary memory (often most of it), not spill: count it as held memory.@@ -340,6 +342,12 @@ class Collector:row.llm = build_view(row.detail, held, row.gpu.shared if snap.shared_is_spill else 0,own or host, log, host=host if own else None,attributed=row.gpu.shown_vram is not None)+ locked = None if own else self.prober.locked_port(ports)+ if locked:+ why = ("rejected the key from the launch arguments" if key else+ "wants an API key, and none was found in the launch arguments (a key set through "+ "LLAMA_API_KEY can't be read from outside the process)")+ row.llm.notes.append(f"Local API on port {locked} {why}. Server details are skipped.")self.inspector.prune(set(procs))return snapdiff --git a/src/gputaskman/llm.py b/src/gputaskman/llm.pyindex 57430c6..4fb4526 100644--- a/src/gputaskman/llm.py+++ b/src/gputaskman/llm.py@@ -16,6 +16,7 @@ import osimport reimport sysimport time+import urllib.errorimport urllib.requestfrom dataclasses import dataclass, field@@ -119,12 +120,68 @@ def ollama_log_path() -> str:_opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))-def _get(url: str, timeout: float = 0.5):+_SECRET_FLAGS = ("--api-key", "--api-key-file")+UNAUTHORIZED = object() # _get result for HTTP 401: the server wants a key+++def api_key(cmd: list[str]) -> str | None:+ """llama-server's API key from its own launch arguments, if it was given one there.++ `--api-key` may list several comma-separated keys; any of them works. `--api-key-file` holds one key per+ line. A key set only through the LLAMA_API_KEY environment variable can't be seen from outside the process.+ The key is only sent back to that process's own localhost port and is never stored or shown.+ """+ for i, arg in enumerate(cmd):+ key, val = arg, None+ if "=" in arg and arg.startswith("--"):+ key, val = arg.split("=", 1)+ if key not in _SECRET_FLAGS:+ continue+ if val is None and i + 1 < len(cmd):+ val = cmd[i + 1]+ if not val:+ continue+ if key == "--api-key":+ return next((k.strip() for k in val.split(",") if k.strip()), None)+ try:+ with open(val, encoding="utf-8", errors="replace") as f:+ lines = f.read(64 << 10).splitlines()+ except OSError:+ continue+ found = next((ln.strip() for ln in lines if ln.strip()), None)+ if found:+ return found+ return None+++def redact_cmdline(cmd: list[str]) -> list[str]:+ """Command line with API keys and key-file paths replaced, for display."""+ out, hide = [], False+ for arg in cmd:+ if hide:+ out.append("***")+ hide = False+ elif arg in _SECRET_FLAGS:+ out.append(arg)+ hide = True+ elif "=" in arg and arg.split("=", 1)[0] in _SECRET_FLAGS:+ out.append(arg.split("=", 1)[0] + "=***")+ else:+ out.append(arg)+ return out+++def _get(url: str, timeout: float = 0.5, key: str | None = None):+ headers = {"Accept": "application/json"}+ if key:+ headers["Authorization"] = f"Bearer {key}"try:- with _opener.open(urllib.request.Request(url, headers={"Accept": "application/json"}), timeout=timeout) as r:+ with _opener.open(urllib.request.Request(url, headers=headers), timeout=timeout) as r:if r.status != 200:return Nonebody = r.read(4 << 20)+ except urllib.error.HTTPError as e:+ return UNAUTHORIZED if e.code == 401 else Noneexcept Exception:return Nonetry:@@ -195,10 +252,16 @@ def lmstudio_models(v1: dict | None, v0: dict | None) -> list[dict] | None:return None-def probe_server(port: int, prev: ServerInfo | None = None) -> ServerInfo | None:- """Read-only GETs. Never calls inference, load or download endpoints."""+def probe_server(port: int, prev: ServerInfo | None = None, key: str | None = None) -> ServerInfo | None:+ """Read-only GETs. Never calls inference, load or download endpoints.++ Returns UNAUTHORIZED when the port answers 401 and no key (or a wrong one) was sent. Probing stops there,+ so a locked server logs one rejected request per probe instead of one per endpoint.+ """base = f"http://127.0.0.1:{port}"- props = _get(base + "/props")+ props = _get(base + "/props", key=key)+ if props is UNAUTHORIZED:+ return UNAUTHORIZEDif isinstance(props, dict) and ("default_generation_settings" in props or "model_path" in props):s = ServerInfo("llama-server", base, model_path=props.get("model_path"))dgs = props.get("default_generation_settings") or {}@@ -207,13 +270,13 @@ def probe_server(port: int, prev: ServerInfo | None = None) -> ServerInfo | Noneif prev and prev.kind == "llama-server" and prev.model_path == s.model_path:s.models = prev.models # identity only changes when the model doeselse:- v1 = _get(base + "/v1/models")+ v1 = _get(base + "/v1/models", key=key)s.models = llama_server_models(v1, s.model_path) if isinstance(v1, dict) else []- slots = _get(base + "/slots")+ slots = _get(base + "/slots", key=key)if isinstance(slots, list):s.slots_total = len(slots)s.slots_busy = sum(1 for x in slots if x.get("is_processing"))- metrics = _get(base + "/metrics")+ metrics = _get(base + "/metrics", key=key)if isinstance(metrics, str):s.gen_tok_s = _metric(metrics, "predicted_tokens_seconds")s.prompt_tok_s = _metric(metrics, "prompt_tokens_seconds")@@ -238,15 +301,21 @@ class ServerProber:def __init__(self) -> None:self._cache: dict[int, tuple[float, ServerInfo | None]] = {}self._last: dict[int, ServerInfo] = {}+ self.locked: set[int] = set() # ports whose last probe was refused with 401- def get(self, ports: list[int]) -> ServerInfo | None:+ def get(self, ports: list[int], key: str | None = None) -> ServerInfo | None:now = time.time()for port in ports:hit = self._cache.get(port)if hit and now - hit[0] < (self.TTL_HIT if hit[1] else self.TTL_MISS):info = hit[1]else:- info = probe_server(port, self._last.get(port))+ info = probe_server(port, self._last.get(port), key)+ if info is UNAUTHORIZED:+ self.locked.add(port)+ info = None+ else:+ self.locked.discard(port)self._cache[port] = (now, info)if info:self._last[port] = info@@ -254,6 +323,10 @@ class ServerProber:return inforeturn None+ def locked_port(self, ports: list[int]) -> int | None:+ """First of these ports whose last probe was refused for a missing or wrong API key."""+ return next((p for p in ports if p in self.locked), None)+# ---------------------------------------------------------------- the breakdowndiff --git a/tests/test_core.py b/tests/test_core.pyindex 4a1e709..ed024eb 100644--- a/tests/test_core.py+++ b/tests/test_core.py@@ -1,6 +1,7 @@from __future__ import annotationsimport contextlib+import dataclassesimport asyncioimport ctypesimport io@@ -27,8 +28,9 @@ from gputaskman.collector import Collector, ProcRow, Snapshot, select_adapter, sfrom gputaskman.dxgi import Adapterfrom gputaskman.gguf import GgufInfo, read_gguffrom gputaskman.handy import HandyEvidence, inspect_handy, _log_cache, _settings_cache-from gputaskman.llm import (LlmView, ServerInfo, build_view, is_gpu_device, llama_server_models, lmstudio_models,- parse_flags, parse_load_log, probe_server)+from gputaskman.llm import (LlmView, ServerInfo, ServerProber, api_key, build_view, is_gpu_device,+ llama_server_models, lmstudio_models, parse_flags, parse_load_log, probe_server,+ redact_cmdline)from gputaskman.nvml import NvDevice, Nvmlfrom gputaskman.pdh import AdapterGpu, GpuCounters, ProcGpufrom gputaskman.procinfo import (AI_KINDS, MappedModel, ProcDetail, ProcInspector, _is_model_file, _match_runtimes,@@ -746,10 +748,68 @@ class IdentityTests(unittest.TestCase):"meta": {"ftype": "Q8_0", "n_ctx": 8192, "size": 447}}]}self.assertEqual(llama_server_models(llama, r"C:\m\Laya-Q8_0.gguf")[0]["type"], "decision")+ def test_api_key_from_launch_arguments(self):+ self.assertEqual(api_key(["llama-server", "--api-key", "abc"]), "abc")+ self.assertEqual(api_key(["llama-server", "--api-key=abc,def"]), "abc")+ self.assertIsNone(api_key(["llama-server", "-m", "x.gguf"]))+ self.assertIsNone(api_key(["llama-server", "--api-key"]))+ with tempfile.TemporaryDirectory() as root:+ path = os.path.join(root, "keys.txt")+ Path(path).write_text("\n file-key \nsecond\n", encoding="utf-8")+ self.assertEqual(api_key(["llama-server", "--api-key-file", path]), "file-key")+ self.assertIsNone(api_key(["llama-server", "--api-key-file", os.path.join(root, "missing")]))+ self.assertEqual(redact_cmdline(["llama-server", "--api-key", "abc", "--api-key-file=/k", "--port", "8080"]),+ ["llama-server", "--api-key", "***", "--api-key-file=***", "--port", "8080"])+ self.assertNotIn("api_key", str(parse_flags(["--api-key", "abc"])))++ def test_llama_server_with_api_key(self):+ """A real local HTTP server that only answers when the bearer token matches, like llama-server --api-key."""+ import http.server+ seen: list[tuple[str, str | None]] = []++ class Handler(http.server.BaseHTTPRequestHandler):+ def do_GET(self):+ auth = self.headers.get("Authorization")+ seen.append((self.path, auth))+ if auth != "Bearer s3cret":+ self.send_response(401)+ self.end_headers()+ return+ body = {"/props": {"model_path": "/m/x.gguf", "total_slots": 2,+ "default_generation_settings": {"n_ctx": 4096}},+ "/slots": [{"is_processing": True}, {"is_processing": False}]}.get(self.path)+ self.send_response(200 if body is not None else 404)+ self.end_headers()+ if body is not None:+ self.wfile.write(json.dumps(body).encode())++ def log_message(self, *args):+ pass++ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ threading.Thread(target=server.serve_forever, daemon=True).start()+ port = server.server_address[1]+ try:+ prober = ServerProber()+ self.assertIsNone(prober.get([port]))+ self.assertEqual(prober.locked, {port})+ self.assertEqual([p for p, _ in seen], ["/props"]) # stops at the first 401, no LM Studio/Ollama probes+ seen.clear()+ s = probe_server(port, key="s3cret")+ self.assertEqual((s.kind, s.n_ctx, s.slots_busy, s.slots_total), ("llama-server", 4096, 1, 2))+ self.assertTrue(all(auth == "Bearer s3cret" for _, auth in seen))+ self.assertNotIn("s3cret", json.dumps(dataclasses.asdict(s)))+ prober._cache.clear()+ self.assertIsNotNone(prober.get([port], "s3cret"))+ self.assertEqual(prober.locked, set())+ finally:+ server.shutdown()+ server.server_close()+def test_lmstudio_falls_back_to_v0_when_v1_is_an_error(self):replies = {"/api/v1/models": {"error": "Unexpected endpoint or method. (GET /api/v1/models)"},"/api/v0/models": {"data": [{"id": "qwen", "state": "loaded", "max_context_length": 8192}]}}- with patch("gputaskman.llm._get", side_effect=lambda url: next(+ with patch("gputaskman.llm._get", side_effect=lambda url, **_: next((v for k, v in replies.items() if url.endswith(k)), None)):s = probe_server(1234)self.assertEqual((s.kind, s.models[0]["name"]), ("lmstudio", "qwen"))