Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python
18.6 KB
Raw Blame
from __future__ import annotations

import contextlib
import io
import json
import os
import tempfile
import unittest
from unittest.mock import patch

from gputaskman.cli import main
from gputaskman.diff import diff, load, match, to_text

GiB = 1 << 30
MiB = 1 << 20


def proc(pid, name, vram=None, shared=0, host=None, ports=(), llm=None, files=(), inferred=None, started=...):
    return {"pid": pid, "host": host, "name": name, "vram_bytes": vram, "shared_bytes": shared,
            "committed_bytes": None, "paged_out_bytes": None, "vram_inferred_bytes": inferred,
            "ports": list(ports), "model_files": [{"path": f, "sources": ["mapped"]} for f in files],
            **({} if started is ... else {"started_at": started}),  # absent: a snapshot from before started_at existed
            **({"llm": llm} if llm else {})}


def pairs(before, after):
    return {(b and b["pid"], a and a["pid"], how) for b, a, how in match(before, after)}


def snap(procs, used, attributed, taken=100.0, gpu="RTX", backend="wddm", adapter=...):
    return {"taken": taken, "gpu": gpu, "backend": backend, "vram_used": used, "shared_used": 0,
            "attributed": attributed, "processes": procs, **({} if adapter is ... else {"adapter": adapter})}


def gpu_adapter(luid=0x4BB7, pci=(1, 0, 0), vendor=0x10DE):
    """The adapter block of a --json snapshot (a dataclass dump: the PCI address is a list)."""
    return {"luid": luid, "name": "RTX", "vendor_id": vendor, "dedicated_vram": 16 * GiB, "shared_system": 0,
            "software": False, "pci_address": list(pci) if pci else None}


def llm(n_ctx, kv, exact=False):
    return {"n_ctx": n_ctx, "segments": [{"label": "weights", "bytes": 4 * GiB, "kind": "weights", "exact": exact},
                                         {"label": "KV", "bytes": kv, "kind": "kv", "exact": exact}]}


class MatchTests(unittest.TestCase):
    def test_pid_then_name_and_ports_then_unique_name(self):
        before = [proc(1, "a.exe"), proc(2, "llama-server", ports=[8080]), proc(3, "solo.exe")]
        after = [proc(1, "a.exe"), proc(20, "llama-server", ports=[8080]), proc(30, "solo.exe")]
        self.assertEqual([(b["pid"], a["pid"], how) for b, a, how in match(before, after)],
                         [(1, 1, "pid"), (2, 20, "name+ports"), (3, 30, "name")])

    def test_reused_pid_ambiguous_names_and_host_pid_space_are_not_paired(self):
        before = [proc(5, "old.exe"), proc(7, "w.exe"), proc(8, "w.exe"), proc(9, "x", host="windows")]
        after = [proc(5, "new.exe"), proc(17, "w.exe"), proc(18, "w.exe"), proc(9, "x")]
        result = {(b and b["pid"], a and a["pid"], how) for b, a, how in match(before, after)}
        self.assertEqual(result, {(None, 5, "new"), (None, 17, "new"), (None, 18, "new"), (None, 9, "new"),
                                  (5, None, "gone"), (7, None, "gone"), (8, None, "gone"), (9, None, "gone")})

    def test_placeholder_rows_and_missing_pids_are_never_paired_or_dropped(self):
        before = [proc(7, "(outside this container)", host="outside"), proc(None, "a"), proc(None, "b"),
                  proc(40, "?", host="windows")]
        after = [proc(9, "(outside this container)", host="outside"), proc(None, "a"), proc(41, "?", host="windows")]
        result = sorted(((b and b["name"], a and a["name"], how) for b, a, how in match(before, after)), key=str)
        self.assertEqual(result, sorted([("(outside this container)", None, "gone"), (None, "(outside this container)", "new"),
                                         ("?", None, "gone"), (None, "?", "new"), ("a", "a", "name"),
                                         ("b", None, "gone")], key=str))
        d = diff(snap(before, 0, 0), snap(after, 0, 0))
        self.assertIn("H:9", to_text(d))
        self.assertIn("W:41", to_text(d))


class PidReuseTests(unittest.TestCase):
    """A PID that comes back with a different start time is another process, however alike the rows look."""

    def test_a_reused_pid_is_gone_and_new_with_or_without_ports(self):
        for ports in ([8080], []):  # name+ports and unique-name fallbacks must not pair them either
            with self.subTest(ports=ports):
                before = [proc(42, "llama-server", ports=ports, started=100.0)]
                after = [proc(42, "llama-server", ports=ports, started=200.0)]
                self.assertEqual(pairs(before, after), {(None, 42, "new"), (42, None, "gone")})

    def test_one_lifetime_still_pairs_by_pid_within_the_rounding_tolerance(self):
        for later in (100.0, 101.9, 102.0, 98.0):
            with self.subTest(later=later):
                self.assertEqual(pairs([proc(42, "a.exe", started=100.0)], [proc(42, "a.exe", started=later)]),
                                 {(42, 42, "pid")})
        self.assertEqual(pairs([proc(42, "a.exe", started=100.0)], [proc(42, "a.exe", started=102.5)]),
                         {(None, 42, "new"), (42, None, "gone")})

    def test_unknown_or_malformed_start_times_keep_the_old_heuristic_pairing(self):
        # ... means the key is absent (an older snapshot); the rest are values a snapshot could carry
        for before_value, after_value in ((..., 200.0), (100.0, ...), (None, None), (None, 200.0), (0.0, 200.0),
                                          (-5.0, 200.0), (True, 200.0), ("100", 200.0), ([100.0], 200.0),
                                          (float("nan"), 200.0), (float("inf"), 200.0)):
            with self.subTest(before=before_value, after=after_value):
                before = [proc(42, "a.exe", started=before_value)]
                after = [proc(42, "a.exe", started=after_value)]
                self.assertEqual(pairs(before, after), {(42, 42, "pid")})
        # no PID means no reuse to speak of: rows without one are still paired by name alone
        self.assertEqual({how for _, _, how in match([proc(None, "a", started=100.0)], [proc(None, "a", started=200.0)])},
                         {"name"})
        before = [proc(42, "llama-server", ports=[8080], started=100.0), proc(50, "solo.exe", started=100.0)]
        after = [proc(77, "llama-server", ports=[8080], started=200.0), proc(60, "solo.exe", started=200.0)]
        self.assertEqual(pairs(before, after), {(42, 77, "name+ports"), (50, 60, "name")})

    def test_a_refused_pair_does_not_make_the_other_rows_less_ambiguous(self):
        before = [proc(42, "llama-server", started=100.0)]
        after = [proc(42, "llama-server", started=200.0), proc(77, "llama-server", started=300.0)]
        self.assertEqual(pairs(before, after), {(None, 42, "new"), (None, 77, "new"), (42, None, "gone")})

    def test_reuse_is_judged_inside_one_pid_namespace(self):
        before = [proc(42, "x", started=100.0), proc(42, "x", host="windows")]
        after = [proc(42, "x", started=200.0), proc(42, "x", host="windows")]
        self.assertEqual(pairs(before, after), {(None, 42, "new"), (42, None, "gone"), (42, 42, "pid")})


class DiffTests(unittest.TestCase):
    def test_path_argument_discovery_does_not_change_measured_memory(self):
        before = snap([proc(1, "custom-server", vram=GiB)], used=GiB, attributed=GiB)
        after = snap([proc(1, "custom-server", vram=GiB)], used=GiB, attributed=GiB)
        after["processes"][0]["model_files"] = [
            {"path": "custom.gguf", "sources": ["path-arg"], "resident": None}]
        result = diff(before, after)
        process = result["processes"][0]
        self.assertEqual(process["model_files"]["added"], [{"path": "custom.gguf", "sources": ["path-arg"]}])
        self.assertEqual(process["measured"]["vram_bytes"]["delta"], 0)
        self.assertEqual(result["device"]["vram_used"]["delta"], 0)
        self.assertIn("[path-arg]", to_text(result))
        self.assertIn("not what is resident in GPU memory", to_text(result))

    def test_measured_runtime_and_mapped_evidence_stay_separate(self):
        before = snap([proc(1, "dwm.exe", vram=2 * GiB),
                       proc(2, "llama-server", vram=5 * GiB, ports=[8080], llm=llm(4096, GiB // 2))],
                      used=8 * GiB, attributed=7 * GiB)
        after = snap([proc(1, "dwm.exe", vram=2 * GiB),
                      proc(22, "llama-server", vram=9 * GiB, ports=[8080], llm=llm(32768, 4 * GiB),
                           files=[r"D:\models\q.gguf"]),
                      proc(3, "python.exe", vram=GiB)],
                     used=13 * GiB, attributed=12 * GiB, taken=142.5)
        d = diff(before, after)
        self.assertEqual(d["seconds"], 42.5)
        self.assertEqual(d["device"]["vram_used"]["delta"], 5 * GiB)
        self.assertEqual(d["device"]["unattributed"], {"before": GiB, "after": GiB, "delta": 0})
        server, new, dwm = d["processes"]
        self.assertEqual((server["match"], server["measured"]["vram_bytes"]["delta"]), ("name+ports", 4 * GiB))
        self.assertEqual(server["runtime"]["n_ctx"], {"before": 4096, "after": 32768, "delta": 28672})
        self.assertEqual(server["runtime"]["segments"]["kv"]["delta"], 3584 * MiB)
        self.assertEqual(server["runtime"]["segments"]["kv"]["source"], "estimated")
        self.assertEqual(server["model_files"]["added"], [{"path": r"D:\models\q.gguf", "sources": ["mapped"]}])
        self.assertEqual(new["match"], "new")
        self.assertEqual(new["measured"]["vram_bytes"], {"before": None, "after": GiB, "delta": None})

        text = to_text(d)
        self.assertIn("2->22", text)
        self.assertIn("context length  4096 -> 32768", text)
        self.assertIn("+3.50 GB  estimated", text)
        self.assertIn("not listed", text)
        self.assertIn("not what is resident in GPU memory", text)
        self.assertNotIn("dwm.exe", text)
        self.assertIn("1 process(es) changed by less than 16 MB", text)

    def test_runtime_reported_segments_and_unknown_measurements(self):
        before = snap([proc(1, "llama", vram=None, llm=llm(4096, GiB, exact=True))], used=None, attributed=None)
        after = snap([proc(1, "llama", vram=None, llm=llm(8192, 2 * GiB, exact=True))], used=None, attributed=None)
        d = diff(before, after)
        self.assertEqual(d["device"]["unattributed"]["delta"], None)
        self.assertEqual(d["processes"][0]["measured"]["vram_bytes"]["delta"], None)
        self.assertEqual(d["processes"][0]["runtime"]["segments"]["kv"]["source"], "runtime-reported")
        self.assertIn("N/A", to_text(d))

    def test_rejects_snapshots_from_different_gpus(self):
        with self.assertRaisesRegex(ValueError, "different GPUs"):
            diff(snap([], 0, 0, gpu="A"), snap([], 0, 0, gpu="B"))

    def test_memory_note_follows_the_snapshots_into_the_comparison(self):
        note = "Intel VRAM used is a sum over the processes gputaskman can inspect."
        noted = {**snap([], 3 * GiB, 0, taken=130.0, backend="drm", gpu="Arc"), "memory_note": note}
        older = snap([], 2 * GiB, 0, backend="drm", gpu="Arc")  # saved by a version that had no note
        for before, after in ((older, noted), (noted, older), (noted, noted)):
            d = diff(before, after)
            self.assertEqual(d["memory_note"], note)
            self.assertIn(note, to_text(d))
        plain = diff(snap([], 2 * GiB, 0), snap([], 3 * GiB, 0))  # every other GPU: nothing to add
        self.assertIsNone(plain["memory_note"])
        self.assertNotIn(note, to_text(plain))

    def test_remainder_segments_follow_the_noise_threshold(self):
        def server(runtime):
            view = {"n_ctx": 4096, "segments": [{"label": "rest", "bytes": runtime, "kind": "runtime", "exact": False}]}
            return proc(1, "llama", vram=GiB + runtime, llm=view)
        d = diff(snap([server(GiB)], 3 * GiB, 2 * GiB), snap([server(GiB + 4096)], 3 * GiB, 2 * GiB))
        self.assertIn("(no process changed by 16 MB or more)", to_text(d))
        self.assertNotIn("llama (1)", to_text(d))

    def test_a_reused_pid_keeps_the_device_change_but_makes_no_process_delta(self):
        before = snap([proc(42, "llama-server", vram=4 * GiB, ports=[8080], started=100.0)], 5 * GiB, 4 * GiB)
        after = snap([proc(42, "llama-server", vram=8 * GiB, ports=[8080], started=200.0)], 9 * GiB, 8 * GiB)
        d = diff(before, after)
        self.assertEqual(d["device"]["vram_used"]["delta"], 4 * GiB)  # the device-wide change is still measured
        self.assertEqual(sorted(p["match"] for p in d["processes"]), ["gone", "new"])
        self.assertTrue(all(p["measured"]["vram_bytes"]["delta"] is None for p in d["processes"]))
        rows = [line for line in to_text(d).splitlines() if line.startswith("llama-server")]
        self.assertEqual(len(rows), 2)  # one row per lifetime, each absent from the other snapshot
        self.assertTrue(all("not listed" in row for row in rows))


class AdapterGuardTests(unittest.TestCase):
    def pair(self, before, after, backend="wddm"):
        return diff(snap([], 0, 0, backend=backend, adapter=before), snap([], 0, 0, backend=backend, adapter=after))

    def test_same_name_but_different_pci_addresses_are_not_one_gpu(self):
        with self.assertRaisesRegex(ValueError, r"different adapters.*'RTX'.*PCI 01:00\.0 vs 02:00\.0"):
            self.pair(gpu_adapter(pci=(1, 0, 0)), gpu_adapter(pci=(2, 0, 0)))

    def test_a_matching_pci_address_survives_a_changed_luid(self):
        d = self.pair(gpu_adapter(luid=0x4BB7), gpu_adapter(luid=0x5A01))  # a reboot, or the adapter was re-enabled
        self.assertEqual((d["adapter_check"], d["adapter_note"]), ("pci", None))
        self.assertNotIn("Warning", to_text(d))

    def test_conflicting_vendors_are_refused_without_pci_addresses(self):
        with self.assertRaisesRegex(ValueError, "vendor 0x10de vs 0x1002"):
            self.pair(gpu_adapter(pci=None, vendor=0x10DE), gpu_adapter(pci=None, vendor=0x1002))

    def test_stable_adapter_keys_separate_identical_gpus_off_windows(self):
        # Linux and macOS keys derive from the PCI slot, so they hold from one snapshot to the next
        with self.assertRaisesRegex(ValueError, "adapter key 0x1000 vs 0x2000"):
            self.pair(gpu_adapter(luid=0x1000, pci=None), gpu_adapter(luid=0x2000, pci=None), backend="nvml")
        d = self.pair(gpu_adapter(luid=0x1000, pci=None), gpu_adapter(luid=0x1000, pci=None), backend="nvml")
        self.assertEqual((d["adapter_check"], d["adapter_note"]), ("key", None))

    def test_a_windows_luid_alone_never_refuses_a_diff(self):
        # Windows reassigns LUIDs at boot: without a PCI address on both sides a changed one proves nothing
        d = self.pair(gpu_adapter(luid=0x4BB7, pci=None), gpu_adapter(luid=0x5A01, pci=None))
        self.assertEqual(d["adapter_check"], "name-only")
        self.assertIn("one GPU or two of the same model", d["adapter_note"])
        self.assertIn("Warning: Their adapter keys differ", to_text(d))
        mixed = self.pair(gpu_adapter(pci=(1, 0, 0)), gpu_adapter(pci=None))  # a PCI address on one side only
        self.assertEqual(mixed["adapter_check"], "key")

    def test_snapshots_with_nothing_to_compare_fall_back_to_name_and_backend_with_a_warning(self):
        legacy = snap([], 0, 0)  # saved by a version that wrote no adapter details
        for before, after in ((legacy, legacy), (legacy, {**legacy, "adapter": gpu_adapter()}),
                              ({**legacy, "adapter": None}, {**legacy, "adapter": None})):
            d = diff(before, after)
            self.assertEqual(d["adapter_check"], "name-only")
            self.assertIn("matched by GPU name and backend only", d["adapter_note"])
            self.assertIn("Warning: These snapshots do not both say which adapter", to_text(d))

    def test_zero_and_malformed_details_are_unknown_not_identities(self):
        unknown = [{"luid": 0, "vendor_id": 0, "pci_address": None},  # zero means unknown, not "adapter 0"
                   {"luid": "4BB7", "vendor_id": True, "pci_address": [1, 0]},  # wrong types, wrong length
                   {"luid": None, "vendor_id": None, "pci_address": [1, 0, -1]}, "RTX", 7, []]
        for backend in ("wddm", "nvml"):  # on nvml a real key mismatch would be refused
            for bad in unknown:
                d = self.pair(bad, gpu_adapter(luid=0x4BB7), backend=backend)
                self.assertEqual(d["adapter_check"], "name-only", (backend, bad))


class LoadAndCliTests(unittest.TestCase):
    def setUp(self):
        self.dir = tempfile.TemporaryDirectory()
        self.addCleanup(self.dir.cleanup)

    def write(self, name, data, encoding):
        path = os.path.join(self.dir.name, name)
        with open(path, "w", encoding=encoding) as f:
            f.write(json.dumps(data))
        return path

    def test_reads_powershell_utf16_and_utf8_bom(self):
        data = snap([proc(1, "a", vram=MiB)], used=MiB, attributed=MiB)
        self.assertEqual(load(self.write("a.json", data, "utf-16")), data)
        self.assertEqual(load(self.write("b.json", data, "utf-8-sig")), data)
        with self.assertRaisesRegex(ValueError, "not a gputaskman --json snapshot"):
            load(self.write("c.json", {"hello": 1}, "utf-8"))

    def test_cli_text_json_and_errors(self):
        a = self.write("a.json", snap([], used=GiB, attributed=GiB), "utf-8")
        b = self.write("b.json", snap([proc(1, "x", vram=GiB)], used=2 * GiB, attributed=2 * GiB), "utf-8")
        out = io.StringIO()
        with patch("sys.argv", ["gputaskman", "diff", a, b]), contextlib.redirect_stdout(out):
            main()
        self.assertIn("dedicated used", out.getvalue())
        out = io.StringIO()
        with patch("sys.argv", ["gputaskman", "diff", a, b, "--json"]), contextlib.redirect_stdout(out):
            main()
        self.assertEqual(json.loads(out.getvalue())["device"]["vram_used"]["delta"], GiB)
        err = io.StringIO()
        with patch("sys.argv", ["gputaskman", "diff", a, os.path.join(self.dir.name, "missing.json")]), \
                contextlib.redirect_stderr(err), self.assertRaises(SystemExit):
            main()
        self.assertIn("missing.json", err.getvalue())

    def test_cli_refuses_conflicting_adapters_and_reports_what_was_compared(self):
        a = self.write("a.json", snap([], 0, 0, adapter=gpu_adapter(pci=(1, 0, 0))), "utf-8")
        b = self.write("b.json", snap([], 0, 0, adapter=gpu_adapter(pci=(2, 0, 0))), "utf-8")
        err = io.StringIO()
        with patch("sys.argv", ["gputaskman", "diff", a, b]), contextlib.redirect_stderr(err), \
                self.assertRaises(SystemExit) as stop:
            main()
        self.assertEqual(stop.exception.code, 2)
        self.assertIn("different adapters", err.getvalue())
        c = self.write("c.json", snap([], 0, 0, adapter=gpu_adapter(luid=0x5A01)), "utf-16")  # same PCI slot, new boot
        out = io.StringIO()
        with patch("sys.argv", ["gputaskman", "diff", a, c, "--json"]), contextlib.redirect_stdout(out):
            main()
        self.assertEqual(json.loads(out.getvalue())["adapter_check"], "pci")


if __name__ == "__main__":
    unittest.main()