Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python
37.4 KB
Raw Blame
"""NVIDIA GPUs in TCC mode: found through NVML (DXGI does not list them) and sampled through NVML.

No TCC GPU was available when this was written. FakeNvml replays what NVIDIA documents: a TCC device answers
nvmlDeviceGetDriverModel with TCC and, unlike a WDDM device, reports each process's memory. The sizes are made up;
what is tested is discovery, ordering, and the rules that keep an unknown value from becoming a zero.
"""

import contextlib
import dataclasses
import io
import sys
import threading
import time
import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch

import gputaskman.collector as collector_mod
from gputaskman import nvml as nvml_mod
from gputaskman import tcc as tcc_mod
from gputaskman.cli import main
from gputaskman.collector import Collector, Snapshot, ordered_adapters, select_adapter
from gputaskman.dxgi import VENDOR_AMD, VENDOR_NVIDIA, Adapter
from gputaskman.nvml import Nvml
from gputaskman.procinfo import ProcDetail
from gputaskman.recovery import LiveCollectors
from gputaskman.report import to_dict, to_text
from gputaskman.tcc import TccAdapter, TccCounters, tcc_adapters

GiB = 1 << 30
MEMORY_V2 = object()  # stands for N.nvmlMemory_v2
NOT_AVAILABLE = (1 << 64) - 1  # NVML_VALUE_NOT_AVAILABLE, as ctypes reports it


def listed_adapter(luid, vendor=VENDOR_NVIDIA, bus=1, total=16 * GiB):
    return Adapter(luid, f"GPU at {bus}", vendor, total, 0, False, (bus, 0, 0))


def tcc_adapter(bus=0x0A, name="Tesla P40", total=24 * GiB, domain=0):
    packed_bdf = bus << 8
    luid = (1 << 62) | domain << 16 | packed_bdf
    return TccAdapter(luid, name, VENDOR_NVIDIA, total, 0, False, (bus, 0, 0), domain)


def device(bus, *, model=1, pending=None, name="Tesla P40", total=24 * GiB, used=0, processes=(), bus_id=None):
    """One NVML device: model 0 is WDDM, 1 is TCC, 2 is MCDM; `processes` are (pid, usedGpuMemory) pairs."""
    return SimpleNamespace(handle=f"handle-{bus:02x}", model=model, pending=pending, name=name, total=total,
                           used=used, processes=list(processes), bus_id=bus_id or f"00000000:{bus:02X}:00.0")


class NvmlError(Exception):
    def __init__(self, value):
        super().__init__(value)
        self.value = value


class FakeNvml:
    """The pynvml calls gputaskman makes, over fixed devices. `log` records every explicit call in order."""

    nvmlMemory_v2 = MEMORY_V2

    def __init__(self):
        self.devices = []
        self.log = []
        self.memory_requests = []
        self.faults = {}  # (function, handle or index) -> the exception that call raises
        self.init_error = self.count_error = None
        self.init_delay = 0.0
        self.v2_rejected = False

    def __getattr__(self, name):  # everything gputaskman reads but these tests do not care about
        if name.startswith("__"):
            raise AttributeError(name)
        return 0 if name.startswith("NVML_") else (lambda *args: None)

    def _call(self, function, key=None):
        self.log.append((function, key))
        error = self.faults.get((function, key))
        if error is not None:
            raise error

    def _device(self, handle):
        for candidate in self.devices:
            if candidate.handle == handle:
                return candidate
        raise NvmlError(2)

    def rehandle(self, candidate, handle):
        """NVML hands out a new handle after it reacquires the device; the old one stops working."""
        candidate.handle = handle

    def nvmlInit(self):
        self._call("nvmlInit")
        if self.init_delay:
            time.sleep(self.init_delay)
        if self.init_error is not None:
            raise self.init_error

    def nvmlShutdown(self):
        self._call("nvmlShutdown")

    def nvmlDeviceGetCount(self):
        self._call("nvmlDeviceGetCount")
        if self.count_error is not None:
            raise self.count_error
        return len(self.devices)

    def nvmlDeviceGetHandleByIndex(self, index):
        self._call("nvmlDeviceGetHandleByIndex", index)
        return self.devices[index].handle

    def nvmlDeviceGetDriverModel(self, handle):
        self._call("nvmlDeviceGetDriverModel", handle)
        candidate = self._device(handle)
        return [candidate.model, candidate.model if candidate.pending is None else candidate.pending]

    def nvmlDeviceGetPciInfo(self, handle):
        self._call("nvmlDeviceGetPciInfo", handle)
        return SimpleNamespace(busId=self._device(handle).bus_id)

    def nvmlDeviceGetName(self, handle):
        self._call("nvmlDeviceGetName", handle)
        return self._device(handle).name

    def nvmlDeviceGetMemoryInfo(self, handle, version=None):
        self.memory_requests.append(version)
        self._call("nvmlDeviceGetMemoryInfo", handle)
        if version is not None and self.v2_rejected:
            raise NvmlError(3)
        candidate = self._device(handle)
        return SimpleNamespace(total=candidate.total, used=candidate.used, free=candidate.total - candidate.used,
                               reserved=0)

    def nvmlDeviceGetComputeRunningProcesses(self, handle):
        self._call("nvmlDeviceGetComputeRunningProcesses", handle)
        return [SimpleNamespace(pid=pid, usedGpuMemory=used) for pid, used in self._device(handle).processes]

    def nvmlSystemGetDriverVersion(self):
        return "581.15"

    def nvmlSystemGetCudaDriverVersion_v2(self):
        return 13000


class TccTestCase(unittest.TestCase):
    def setUp(self):
        self.api = FakeNvml()
        for target, value in (("gputaskman.nvml.N", self.api), ("gputaskman.nvml._unsafe", False),
                              ("gputaskman.nvml.LOST", set()), ("gputaskman.tcc._found", None),
                              ("gputaskman.tcc._discovery_status", "")):
            patcher = patch(target, value)
            patcher.start()
            self.addCleanup(patcher.stop)

    def names(self):
        return [name for name, _key in self.api.log]

    def counters(self, adapter=None):
        """A TccCounters over a real Nvml session on the fake, with the setup calls cleared from the log."""
        nvml = Nvml()
        self.api.log.clear()
        self.api.memory_requests.clear()
        return TccCounters(adapter or tcc_adapter(), nvml), nvml


class DiscoveryTests(TccTestCase):
    def test_finds_a_tcc_gpu_with_every_field(self):
        self.api.devices = [device(0x0A)]
        (found,) = tcc_adapters([])
        self.assertIsInstance(found, TccAdapter)
        self.assertEqual((found.name, found.vendor_id, found.vendor, found.dedicated_vram, found.shared_system,
                          found.software, found.pci_address),
                         ("Tesla P40", VENDOR_NVIDIA, "NVIDIA", 24 * GiB, 0, False, (0x0A, 0, 0)))

    def test_only_a_gpu_that_is_in_tcc_mode_now_counts(self):
        # WDDM, MCDM, an unknown model, a WDDM GPU that will switch to TCC at the next reboot, and the reverse.
        self.api.devices = [device(1, model=0), device(2, model=2), device(3, model=7),
                            device(4, model=0, pending=1), device(5, model=1, pending=0)]
        self.assertEqual([a.pci_address[0] for a in tcc_adapters([])], [5])

    def test_a_gpu_dxgi_lists_is_never_added_again(self):
        self.api.devices = [device(0x0A), device(0x0B)]
        listed = Adapter(7, "Tesla P40", VENDOR_NVIDIA, 24 * GiB, 0, False, (0x0A, 0, 0))
        self.assertEqual([a.pci_address for a in tcc_adapters([listed])], [(0x0B, 0, 0)])

    def test_listed_adapters_that_cannot_be_paired_hide_nothing(self):
        self.api.devices = [device(0x0A)]
        unpaired = Adapter(7, "Tesla P40", VENDOR_NVIDIA, 24 * GiB, 0, False)
        elsewhere = Adapter(8, "Tesla P40", VENDOR_NVIDIA, 24 * GiB, 0, False, (0x0B, 0, 0))
        self.assertEqual(len(tcc_adapters([unpaired, elsewhere])), 1)

    def test_a_nonzero_pci_domain_is_preserved_in_tcc_identity(self):
        self.api.devices = [device(1, bus_id="00010000:01:00.0"), device(2)]
        found = tcc_adapters([])
        self.assertEqual([(a.pci_domain, a.pci_address) for a in found], [(0x10000, (1, 0, 0)), (0, (2, 0, 0))])
        self.assertNotEqual(found[0].luid, found[1].luid)

    def test_identical_bdf_values_in_different_domains_have_distinct_ids(self):
        self.api.devices = [device(1, bus_id="00000000:01:00.0"), device(2, bus_id="00010000:01:00.0")]
        found = tcc_adapters([])
        self.assertEqual([(a.pci_domain, a.pci_address) for a in found],
                         [(0, (1, 0, 0)), (0x10000, (1, 0, 0))])
        self.assertEqual(len({a.luid for a in found}), 2)

    def test_only_domain_zero_can_be_deduplicated_against_dxgi(self):
        self.api.devices = [device(1, bus_id="00000000:01:00.0"), device(2, bus_id="00010000:01:00.0")]
        listed = Adapter(7, "Listed GPU", VENDOR_NVIDIA, 24 * GiB, 0, False, (1, 0, 0))
        found = tcc_adapters([listed])
        self.assertEqual([(a.pci_domain, a.pci_address) for a in found], [(0x10000, (1, 0, 0))])

    def test_bytes_and_lowercase_text_from_other_bindings(self):
        self.api.devices = [device(0x0A, bus_id=b"00000000:0a:00.0", name=b"Tesla P40")]
        (found,) = tcc_adapters([])
        self.assertEqual((found.name, found.pci_address), ("Tesla P40", (0x0A, 0, 0)))

    def test_never_raises_without_pynvml(self):
        with patch("gputaskman.nvml.N", None):
            self.assertEqual(tcc_adapters([]), [])
        self.assertEqual(self.api.log, [])

    def test_nvml_that_cannot_start_means_no_tcc_gpus_and_no_shutdown(self):
        self.api.devices = [device(0x0A)]
        self.api.init_error = NvmlError(9)
        self.assertEqual(tcc_adapters([]), [])
        self.assertEqual(self.api.log, [("nvmlInit", None)])

    def test_a_failing_device_count_is_survivable(self):
        self.api.devices = [device(0x0A)]
        for error, unsafe in ((NvmlError(999), False), (OSError("access violation"), True)):
            with self.subTest(error=type(error).__name__):
                self.api.log.clear()
                self.api.count_error = error
                tcc_mod._found = None
                nvml_mod._unsafe = False
                self.assertEqual(tcc_adapters([]), [])
                self.assertIs(nvml_mod._unsafe, unsafe)
                self.assertEqual(self.names().count("nvmlShutdown"), 0 if unsafe else 1)
                self.assertIn("unsafe driver fault" if unsafe else "incomplete", tcc_mod.tcc_discovery_status())

    def test_one_unreadable_device_does_not_hide_a_tcc_peer(self):
        self.api.devices = [device(1), device(2)]
        for key, error, unsafe in (
                (("nvmlDeviceGetHandleByIndex", 0), NvmlError(15), False),
                (("nvmlDeviceGetDriverModel", "handle-01"), NvmlError(3), False),
                (("nvmlDeviceGetMemoryInfo", "handle-01"), NvmlError(3), False),
                (("nvmlDeviceGetPciInfo", "handle-01"), OSError("access violation"), True)):
            with self.subTest(call=key[0]):
                self.api.faults = {key: error}
                self.api.log.clear()
                tcc_mod._found = None
                nvml_mod._unsafe = False
                self.assertEqual([a.pci_address for a in tcc_adapters([])], [(2, 0, 0)])
                self.assertIs(nvml_mod._unsafe, unsafe)  # an access violation inside nvml.dll can hang a shutdown
                self.assertEqual(self.names().count("nvmlShutdown"), 0 if unsafe else 1)
                self.assertIn("unsafe driver fault" if unsafe else "incomplete", tcc_mod.tcc_discovery_status())

    def test_nvml_is_asked_once_per_process_even_when_it_found_nothing(self):
        for devices in ([device(0x0A)], []):
            with self.subTest(devices=len(devices)):
                self.api.devices = devices
                self.api.log.clear()
                tcc_mod._found = None
                first = tcc_adapters([])
                self.assertEqual(tcc_adapters([]), first)
                self.assertEqual(self.names().count("nvmlInit"), 1)

    def test_init_comes_first_and_exactly_one_shutdown_comes_last(self):
        self.api.devices = [device(1), device(2, model=0), device(3)]
        tcc_adapters([])
        names = self.names()
        self.assertEqual((names[0], names[-1]), ("nvmlInit", "nvmlShutdown"))
        self.assertEqual((names.count("nvmlInit"), names.count("nvmlShutdown")), (1, 1))

    def test_no_shutdown_once_nvml_is_marked_unsafe(self):
        self.api.devices = [device(0x0A)]
        nvml_mod._unsafe = True
        self.assertEqual(tcc_adapters([]), [])
        self.assertEqual(self.api.log, [])
        self.assertIn("marked unsafe", tcc_mod.tcc_discovery_status())

    def test_nvml_sessions_make_no_dll_calls_after_the_unsafe_flag_is_set(self):
        nvml_mod._unsafe = True
        nvml = Nvml()
        nvml.refresh()
        self.assertEqual(nvml.sample(), [])
        nvml.close()
        self.assertEqual(self.api.log, [])

    def test_discovery_status_distinguishes_no_tcc_devices_from_a_failed_probe(self):
        self.assertEqual(tcc_adapters([]), [])
        self.assertIn("succeeded; no TCC GPUs found", tcc_mod.tcc_discovery_status())
        tcc_mod._found = None
        self.api.init_error = NvmlError("private driver detail")
        self.assertEqual(tcc_adapters([]), [])
        self.assertIn("initialization failed", tcc_mod.tcc_discovery_status())
        self.assertNotIn("private driver detail", tcc_mod.tcc_discovery_status())

    def test_a_failing_shutdown_is_reported_without_hiding_discovery(self):
        self.api.devices = [device(0x0A)]
        self.api.faults = {("nvmlShutdown", None): NvmlError(3)}
        self.assertEqual(len(tcc_adapters([])), 1)
        self.assertIn("shutdown failed", tcc_mod.tcc_discovery_status())

    def test_the_listed_adapters_filter_every_call_while_nvml_is_asked_once(self):
        self.api.devices = [device(0x0A)]
        listed = Adapter(7, "Tesla P40", VENDOR_NVIDIA, 24 * GiB, 0, False, (0x0A, 0, 0))
        self.assertEqual(len(tcc_adapters([])), 1)
        self.assertEqual(tcc_adapters([listed]), [])
        self.assertEqual(len(tcc_adapters([])), 1)
        self.assertEqual(self.names().count("nvmlInit"), 1)

    def test_concurrent_first_calls_ask_nvml_once(self):
        self.api.devices = [device(0x0A)]
        self.api.init_delay = 0.05
        barrier, results = threading.Barrier(6), []

        def ask():
            barrier.wait()
            results.append(len(tcc_adapters([])))

        threads = [threading.Thread(target=ask, daemon=True) for _ in range(6)]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join(10)
        self.assertEqual(results, [1] * 6)
        self.assertEqual(self.names().count("nvmlInit"), 1)

    def test_luids_are_made_from_the_pci_address_and_never_change(self):
        self.api.devices = [device(0x3B, bus_id="00000000:3B:02.1"), device(0x0A)]
        first = tcc_adapters([])
        tcc_mod._found = None
        again = tcc_adapters([])
        self.assertEqual([a.luid for a in first], [a.luid for a in again])
        self.assertEqual([a.luid for a in first], [(1 << 62) | 0x3B << 8 | 2 << 3 | 1,
                                                   (1 << 62) | 0x0A << 8])
        self.assertTrue(all(a.luid >= 1 << 62 for a in first))  # far from the small LUIDs DXGI hands out


class OrderingTests(unittest.TestCase):
    def test_tcc_gpus_sort_after_every_listed_adapter_so_indexes_never_shift(self):
        listed = [listed_adapter(1, 0x8086, 1, 2 * GiB), listed_adapter(2, VENDOR_NVIDIA, 2, 16 * GiB),
                  listed_adapter(3, VENDOR_AMD, 3, 8 * GiB)]
        tcc = tcc_adapter(total=48 * GiB)  # more memory than any of them
        ordered = ordered_adapters(listed + [tcc])
        self.assertEqual(ordered[:-1], ordered_adapters(listed))
        self.assertIs(ordered[-1], tcc)

    def test_two_tcc_gpus_sort_by_memory_then_luid(self):
        small, big, twin = tcc_adapter(0x0A, total=12 * GiB), tcc_adapter(0x0B, total=24 * GiB), tcc_adapter(0x0C, total=12 * GiB)
        self.assertEqual(ordered_adapters([small, big]), [big, small])
        self.assertEqual(ordered_adapters([twin, small]), [small, twin])

    def test_a_machine_with_only_a_tcc_gpu_selects_it(self):
        tcc = tcc_adapter()
        self.assertIs(select_adapter([tcc], 0), tcc)

    def test_gpu_index_after_the_listed_adapters_selects_the_tcc_gpu(self):
        listed, tcc = listed_adapter(1), tcc_adapter()
        self.assertIs(select_adapter([tcc, listed], 0), listed)
        self.assertIs(select_adapter([tcc, listed], 1), tcc)
        with self.assertRaisesRegex(ValueError, "out of range; 2 GPU adapter"):
            select_adapter([tcc, listed], 2)

    def test_json_describes_a_tcc_adapter_like_any_other(self):
        self.assertEqual(set(dataclasses.asdict(tcc_adapter())), set(dataclasses.asdict(listed_adapter(1))))


class CounterTests(TccTestCase):
    def test_reports_nvml_memory_for_the_adapter_and_each_process(self):
        self.api.devices = [device(0x0A, used=9 * GiB, processes=[(100, 6 * GiB), (200, 3 * GiB)])]
        counters, _ = self.counters()
        per_luid, adapters = counters.sample()
        luid = counters.adapter.luid
        self.assertEqual(list(per_luid), [luid])
        self.assertEqual({pid: g.dedicated for pid, g in per_luid[luid].items()}, {100: 6 * GiB, 200: 3 * GiB})
        for g in per_luid[luid].values():
            self.assertEqual(g.available, {"dedicated"})  # no shared, commit or engine counters exist for TCC
            self.assertTrue(g.attributed)
        self.assertEqual((adapters[luid].dedicated, adapters[luid].shared), (9 * GiB, None))

    def test_the_adapter_discovery_returns_is_the_one_the_counters_find(self):
        self.api.devices = [device(0x3B, bus_id="00000000:3B:02.1", used=GiB, processes=[(5, GiB)])]
        (found,) = tcc_adapters([])
        per_luid, adapters = TccCounters(found, Nvml()).sample()
        self.assertEqual((list(per_luid), list(adapters)), ([found.luid], [found.luid]))
        self.assertEqual(per_luid[found.luid][5].dedicated, GiB)

    def test_counters_locate_the_gpu_by_domain_when_bdf_values_match(self):
        self.api.devices = [device(1, bus_id="00000000:01:00.0", used=GiB, processes=[(5, GiB)]),
                            device(2, bus_id="00010000:01:00.0", used=2 * GiB, processes=[(6, GiB)])]
        counters = TccCounters(tcc_adapter(1, domain=0x10000), Nvml())
        per_luid, adapters = counters.sample()
        self.assertEqual(list(per_luid[counters.adapter.luid]), [6])
        self.assertEqual(adapters[counters.adapter.luid].dedicated, 2 * GiB)

    def test_memory_uses_the_v2_query_and_falls_back_to_the_plain_one(self):
        self.api.devices = [device(0x0A, used=GiB)]
        counters, _ = self.counters()
        counters.sample()
        self.assertEqual(self.api.memory_requests, [MEMORY_V2])
        self.api.memory_requests.clear()
        self.api.v2_rejected = True
        _, adapters = counters.sample()
        self.assertEqual(self.api.memory_requests, [MEMORY_V2, None])
        self.assertEqual(adapters[counters.adapter.luid].dedicated, GiB)

    def test_hidden_process_memory_is_unknown_not_zero(self):
        for hidden in (None, NOT_AVAILABLE, 1 << 63):
            with self.subTest(value=hidden):
                self.api.devices = [device(0x0A, used=GiB, processes=[(100, hidden)])]
                counters, _ = self.counters()
                (g,) = counters.sample()[0][counters.adapter.luid].values()
                self.assertFalse(g.attributed)
                self.assertEqual(g.available, set())
                self.assertIsNone(g.shown_vram)

    def test_a_measured_zero_and_the_largest_signed_value_stay_measured(self):
        self.api.devices = [device(0x0A, processes=[(100, 0), (200, (1 << 63) - 1)])]
        counters, _ = self.counters()
        procs = counters.sample()[0][counters.adapter.luid]
        self.assertEqual([(g.attributed, g.dedicated, g.available) for g in (procs[100], procs[200])],
                         [(True, 0.0, {"dedicated"}), (True, float((1 << 63) - 1), {"dedicated"})])

    def test_a_process_listed_twice_is_counted_once_at_its_larger_size(self):
        for entries in ([(100, 2 * GiB), (100, 3 * GiB)], [(100, 3 * GiB), (100, 2 * GiB)]):
            with self.subTest(entries=entries):
                self.api.devices = [device(0x0A, processes=entries)]
                counters, _ = self.counters()
                (g,) = counters.sample()[0][counters.adapter.luid].values()
                self.assertEqual(g.dedicated, 3 * GiB)

    def test_one_unknown_entry_makes_a_process_unknown_in_either_order(self):
        for entries in ([(100, 2 * GiB), (100, None)], [(100, None), (100, 2 * GiB)]):
            with self.subTest(entries=entries):
                self.api.devices = [device(0x0A, processes=entries)]
                counters, _ = self.counters()
                (g,) = counters.sample()[0][counters.adapter.luid].values()
                self.assertFalse(g.attributed)
                self.assertIsNone(g.shown_vram)

    def test_a_gpu_with_no_processes_reports_an_empty_set_not_unavailable(self):
        self.api.devices = [device(0x0A, used=GiB)]
        counters, _ = self.counters()
        per_luid, adapters = counters.sample()
        self.assertEqual(per_luid, {counters.adapter.luid: {}})
        self.assertEqual(adapters[counters.adapter.luid].dedicated, GiB)

    def test_unreadable_is_reported_as_nothing_never_as_a_partial_guess(self):
        for case in ("nvml did not start", "gpu is not in the handle list", "process list failed", "memory failed"):
            with self.subTest(case=case):
                self.api.devices = [device(0x0B if case == "gpu is not in the handle list" else 0x0A,
                                           used=GiB, processes=[(100, GiB)])]
                self.api.faults = {}
                self.api.init_error = NvmlError(9) if case == "nvml did not start" else None
                if case == "process list failed":
                    self.api.faults = {("nvmlDeviceGetComputeRunningProcesses", "handle-0a"): NvmlError(3)}
                if case == "memory failed":  # both the v2 and the plain query fail
                    self.api.faults = {("nvmlDeviceGetMemoryInfo", "handle-0a"): NvmlError(3)}
                counters, _ = self.counters()
                self.assertEqual(counters.sample(), ({}, {}))
                self.assertEqual(nvml_mod.LOST, set())  # an NVML error code is not a fault inside nvml.dll
                self.assertFalse(nvml_mod._unsafe)
                if case != "process list failed" and case != "memory failed":
                    self.assertNotIn("nvmlDeviceGetMemoryInfo", self.names())
                    self.assertNotIn("nvmlDeviceGetComputeRunningProcesses", self.names())

    def test_a_fault_inside_nvml_stops_reading_this_gpu_for_good(self):
        for function in ("nvmlDeviceGetMemoryInfo", "nvmlDeviceGetComputeRunningProcesses"):
            with self.subTest(function=function):
                nvml_mod.LOST.clear()
                nvml_mod._unsafe = False
                self.api.devices = [device(0x0A, used=GiB, processes=[(100, GiB)])]
                self.api.faults = {(function, "handle-0a"): OSError("access violation")}
                counters, _ = self.counters()
                self.assertEqual(counters.sample(), ({}, {}))
                self.assertEqual(nvml_mod.LOST, {"00000000:0A:00.0"})
                self.assertTrue(nvml_mod._unsafe)
                self.api.log.clear()
                self.api.faults = {}
                self.assertEqual(counters.sample(), ({}, {}))
                self.assertEqual(self.api.log, [])  # not one more read of a GPU whose reads faulted

    def test_a_bus_nvml_already_lost_is_matched_by_address_not_by_text(self):
        nvml_mod.LOST.add("00000000:0a:00.0")  # another reader's spelling of the same address
        self.api.devices = [device(0x0A, used=GiB, processes=[(100, GiB)])]
        counters, _ = self.counters()
        self.assertEqual(counters.sample(), ({}, {}))
        self.assertEqual(self.names(), ["nvmlDeviceGetPciInfo"])

    def test_a_fault_flag_left_by_another_gpu_does_not_mark_this_one_lost(self):
        self.api.devices = [device(0x0A, used=GiB)]
        counters, nvml = self.counters()
        nvml._faulted = True  # Nvml.sample() sets it while it reads some other GPU
        per_luid, _ = counters.sample()
        self.assertEqual(per_luid, {counters.adapter.luid: {}})
        self.assertEqual(nvml_mod.LOST, set())
        self.assertFalse(nvml_mod._unsafe)

    def test_it_finds_its_handle_again_after_nvml_reacquires_handles(self):
        self.api.devices = [device(0x0A, used=GiB)]
        counters, nvml = self.counters()
        self.assertIn(counters.adapter.luid, counters.sample()[1])
        self.api.rehandle(self.api.devices[0], "handle-new")
        self.assertEqual(counters.sample(), ({}, {}))  # the old handle no longer works
        nvml.refresh()
        self.assertIn(counters.adapter.luid, counters.sample()[1])

    def test_a_failed_pci_lookup_is_retried_on_the_next_sample(self):
        self.api.devices = [device(0x0A, used=GiB)]
        counters, _ = self.counters()
        self.api.faults = {("nvmlDeviceGetPciInfo", "handle-0a"): NvmlError(3)}
        self.assertEqual(counters.sample(), ({}, {}))
        self.api.faults = {}
        self.assertIn(counters.adapter.luid, counters.sample()[1])

    def test_the_pci_address_is_read_once(self):
        self.api.devices = [device(0x0A, used=GiB)]
        counters, _ = self.counters()
        counters.sample()
        counters.sample()
        self.assertEqual(self.names().count("nvmlDeviceGetPciInfo"), 1)

    def test_close_never_touches_the_collectors_nvml_session(self):
        self.api.devices = [device(0x0A)]
        counters, nvml = self.counters()
        counters.close()
        counters.close()
        self.assertTrue(nvml.ok)
        self.assertEqual(self.api.log, [])

    def test_nothing_is_read_once_the_nvml_session_is_closed(self):
        self.api.devices = [device(0x0A, used=GiB, processes=[(100, GiB)])]
        counters, nvml = self.counters()
        self.assertIn(counters.adapter.luid, counters.sample()[1])
        nvml.close()  # keeps its old handles, which are not valid after nvmlShutdown
        self.api.log.clear()
        self.assertEqual(counters.sample(), ({}, {}))
        self.assertEqual(self.api.log, [])

    def test_each_gpu_sees_only_its_own_processes(self):
        self.api.devices = [device(1, model=0, processes=[(10, GiB)]),
                            device(0x0A, used=3 * GiB, processes=[(20, 2 * GiB)]),
                            device(0x0B, used=5 * GiB, processes=[(30, 4 * GiB)])]
        nvml = Nvml()
        first, second = TccCounters(tcc_adapter(0x0A), nvml), TccCounters(tcc_adapter(0x0B), nvml)
        for counters, pid, used, memory in ((first, 20, 2 * GiB, 3 * GiB), (second, 30, 4 * GiB, 5 * GiB)):
            per_luid, adapters = counters.sample()
            luid = counters.adapter.luid
            self.assertEqual({p: g.dedicated for p, g in per_luid[luid].items()}, {pid: used})
            self.assertEqual(adapters[luid].dedicated, memory)


class CollectorTests(TccTestCase):
    def collector(self, adapter=None, *, listed=()):
        adapter = adapter or tcc_adapter()
        c = Collector.__new__(Collector)
        c.adapter = adapter
        c.adapters = [*listed, adapter]
        c.gpu_index, c.scan_budget, c.host, c.host_status = len(listed), None, None, ""
        c.nvml = Nvml()
        c.counters = TccCounters(adapter, c.nvml)
        c.adl, c.amdsmi = Mock(), Mock()
        c.inspector, c.prober = Mock(), Mock()
        c.inspector.get.side_effect = lambda pid: ProcDetail(pid, f"app{pid}.exe")
        c.inspector.needs_scan.return_value = False
        return c

    def sample(self, c):
        with patch("gputaskman.collector.BACKEND", "wddm"), patch("gputaskman.collector.sys.platform", "win32"):
            return c.sample()

    def test_a_tcc_gpu_gets_its_memory_and_processes_from_nvml(self):
        self.api.devices = [device(0x0A, used=9 * GiB, processes=[(200, 2 * GiB), (100, 6 * GiB)])]
        snap = self.sample(self.collector())
        self.assertEqual((snap.nv.driver_model, snap.nv.mem_total, snap.vram_total), ("TCC", 24 * GiB, 24 * GiB))
        self.assertEqual((snap.driver, snap.cuda, snap.backend), ("581.15", "13.0", "wddm"))
        self.assertEqual((snap.vram_used, snap.shared_used), (9 * GiB, None))
        self.assertTrue(snap.counter_adapter_matched)
        self.assertIn("TCC compute-only", snap.counter_status)
        self.assertEqual([r.pid for r in snap.procs], [100, 200])  # largest first
        self.assertEqual(snap.attributed, 8 * GiB)
        data = to_dict(snap)
        self.assertEqual({p["pid"]: p["vram_bytes"] for p in data["processes"]}, {100: 6 * GiB, 200: 2 * GiB})
        for process in data["processes"]:
            for key in ("shared_bytes", "committed_bytes", "paged_out_bytes", "gpu_pct", "engines"):
                self.assertIsNone(process[key], key)  # no such counters exist for a TCC GPU: unknown, not zero
        self.assertIsNone(data["shared_used"])
        self.assertIn("TCC compute-only", to_text(snap))

    def test_it_is_told_apart_from_a_wddm_gpu_in_the_same_machine(self):
        self.api.devices = [device(1, model=0, name="GTX 1080 Ti", total=11 * GiB, used=2 * GiB),
                            device(0x0A, name="P102-100", total=10 * GiB, used=GiB, processes=[(7, GiB)])]
        snap = self.sample(self.collector(tcc_adapter(0x0A, "P102-100", 10 * GiB), listed=[listed_adapter(1, bus=1)]))
        self.assertEqual((snap.nv.name, snap.nv.driver_model, snap.vram_used), ("P102-100", "TCC", GiB))
        self.assertEqual([(r.pid, r.gpu.dedicated) for r in snap.procs], [(7, GiB)])

    def test_device_telemetry_matches_tcc_adapter_by_domain(self):
        self.api.devices = [device(1, bus_id="00010000:01:00.0", total=24 * GiB, used=GiB)]
        snap = self.sample(self.collector(tcc_adapter(1, domain=0x10000)))
        self.assertEqual((snap.nv.driver_model, snap.nv.mem_total, snap.vram_used),
                         ("TCC", 24 * GiB, GiB))

    def test_hidden_process_memory_leaves_the_total_unknown_but_the_adapter_measured(self):
        self.api.devices = [device(0x0A, used=9 * GiB, processes=[(100, 6 * GiB), (200, None)])]
        snap = self.sample(self.collector())
        self.assertIsNone(snap.attributed)
        self.assertEqual(snap.vram_used, 9 * GiB)
        data = to_dict(snap)
        self.assertEqual({p["pid"]: p["vram_bytes"] for p in data["processes"]}, {100: 6 * GiB, 200: None})
        self.assertIsNone(data["attributed"])

    def test_unreadable_nvml_is_explained_not_guessed(self):
        self.api.init_error = NvmlError(9)
        c = self.collector()
        snap = self.sample(c)
        self.assertFalse(snap.counter_adapter_matched)
        self.assertIn("NVML could not read this TCC GPU", snap.counter_status)
        self.assertEqual(snap.procs, [])
        self.assertIsNone(snap.vram_used)
        self.assertIsNone(snap.attributed)
        self.assertEqual(snap.vram_total, c.adapter.dedicated_vram)

    def test_a_lost_gpu_says_to_restart(self):
        nvml_mod.LOST.add("00000000:0A:00.0")
        self.api.devices = [device(0x0A, used=GiB)]
        snap = self.sample(self.collector())
        self.assertTrue(snap.nv_lost)
        self.assertIn("restart gputaskman", snap.counter_status)
        self.assertIsNone(snap.vram_used)


class CollectorSetupTests(TccTestCase):
    NAMES = ("GpuCounters", "Nvml", "AmdSmi", "Adl", "ProcInspector", "ServerProber")

    def mocks(self, stack):
        mocks = {name: stack.enter_context(patch(f"gputaskman.collector.{name}")) for name in self.NAMES}
        stack.enter_context(patch("gputaskman.collector.BACKEND", "wddm"))
        return mocks

    def test_a_tcc_gpu_never_creates_windows_counters(self):
        with contextlib.ExitStack() as stack:
            mocks = self.mocks(stack)
            c = Collector(0, adapters=[tcc_adapter()], host=False)
        mocks["GpuCounters"].assert_not_called()
        mocks["Nvml"].assert_called_once_with()  # one NVML session, shared with the counters
        self.assertIsInstance(c.counters, TccCounters)
        self.assertIs(c.counters.nvml, mocks["Nvml"].return_value)
        self.assertIs(c.nvml, mocks["Nvml"].return_value)

    def test_a_listed_nvidia_gpu_still_uses_windows_counters(self):
        with contextlib.ExitStack() as stack:
            mocks = self.mocks(stack)
            c = Collector(0, adapters=[listed_adapter(1)], host=False)
        mocks["GpuCounters"].assert_called_once_with()
        self.assertIs(c.counters, mocks["GpuCounters"].return_value)

    def test_a_failure_while_building_closes_the_nvml_session_it_opened(self):
        with contextlib.ExitStack() as stack:
            mocks = self.mocks(stack)
            mocks["Adl"].side_effect = OSError("no ADL")
            with self.assertRaises(OSError):
                Collector(0, adapters=[tcc_adapter()], host=False)
        mocks["Nvml"].return_value.close.assert_called_once()
        mocks["GpuCounters"].assert_not_called()


class CliTests(unittest.TestCase):
    def test_list_gpus_shows_a_tcc_gpu_by_pci_address_after_the_listed_ones(self):
        output = io.StringIO()
        with patch("sys.argv", ["gputaskman", "--list-gpus"]), \
                patch("gputaskman.collector.enumerate_adapters",
                      return_value=[tcc_adapter(0x0A), listed_adapter(0x1234ABCD)]), \
                patch("gputaskman.tcc.tcc_discovery_status", return_value=""), \
                contextlib.redirect_stdout(output):
            main()
        self.assertEqual(output.getvalue().splitlines(), [
            "0: GPU at 1 (NVIDIA, LUID 0x000000001234abcd)",
            "1: Tesla P40 (NVIDIA, TCC compute-only, PCI 0a:00.0)"])

    def test_list_gpus_reports_nvml_discovery_status_on_stderr(self):
        output, error = io.StringIO(), io.StringIO()
        with patch("sys.argv", ["gputaskman", "--list-gpus"]), \
                patch("gputaskman.collector.enumerate_adapters", return_value=[]), \
                patch("gputaskman.tcc.tcc_discovery_status",
                      return_value="NVML initialization failed; TCC GPUs could not be checked."), \
                contextlib.redirect_stdout(output), contextlib.redirect_stderr(error):
            main()
        self.assertEqual(output.getvalue(), "")
        self.assertEqual(error.getvalue().strip(),
                         "TCC discovery: NVML initialization failed; TCC GPUs could not be checked.")

    def test_list_gpus_displays_full_domain_for_tcc_gpu(self):
        output = io.StringIO()
        with patch("sys.argv", ["gputaskman", "--list-gpus"]), \
                patch("gputaskman.collector.enumerate_adapters", return_value=[tcc_adapter(1, domain=0x10000)]), \
                patch("gputaskman.tcc.tcc_discovery_status", return_value=""), \
                contextlib.redirect_stdout(output):
            main()
        self.assertEqual(output.getvalue().strip(),
                         "0: Tesla P40 (NVIDIA, TCC compute-only, PCI 00010000:01:00.0)")


@unittest.skipUnless(sys.platform == "win32", "the DXGI wrapper exists only on Windows")
class WindowsEnumerationTests(unittest.TestCase):
    def test_tcc_gpus_come_after_the_adapters_dxgi_lists(self):
        listed, tcc = [listed_adapter(1)], tcc_adapter()
        with patch("gputaskman.collector.dxgi.enumerate_adapters", return_value=listed) as dxgi, \
                patch("gputaskman.collector.tcc_adapters", return_value=[tcc]) as found:
            self.assertEqual(collector_mod.enumerate_adapters(strict=True), [*listed, tcc])
        dxgi.assert_called_once_with(strict=True)
        found.assert_called_once_with(listed)

    def test_without_tcc_gpus_the_dxgi_list_is_returned_unchanged(self):
        listed = [listed_adapter(1)]
        with patch("gputaskman.collector.dxgi.enumerate_adapters", return_value=listed), \
                patch("gputaskman.collector.tcc_adapters", return_value=[]):
            self.assertEqual(collector_mod.enumerate_adapters(), listed)

    def test_a_strict_dxgi_failure_propagates_before_nvml_is_asked(self):
        with patch("gputaskman.collector.dxgi.enumerate_adapters", side_effect=OSError("DXGI failed")), \
                patch("gputaskman.collector.tcc_adapters") as found:
            with self.assertRaises(OSError):
                collector_mod.enumerate_adapters(strict=True)
        found.assert_not_called()


class RecoveryTests(TccTestCase):
    def test_a_tcc_slot_keeps_its_collector_while_a_listed_gpu_is_re_enabled(self):
        wddm, tcc = listed_adapter(1), tcc_adapter()
        topology, created = [tcc, wddm], []

        def factory(index, *, adapters, scan_budget):
            adapter = adapters[index]
            collector = Mock(adapter=adapter, scan_budget=scan_budget)
            collector.sample.side_effect = lambda: Snapshot(1, adapter, adapters, None, "", "", adapter.dedicated_vram,
                                                            0, 0, counter_adapter_matched=True)
            created.append(collector)
            return collector

        live = LiveCollectors(None, collector_factory=factory, discover=lambda **_kwargs: list(topology))
        self.addCleanup(live.close)
        self.assertEqual([slot.adapter for slot in live.slots], [wddm, tcc])  # TCC last
        tcc_collector = created[1]
        topology[:] = [tcc, listed_adapter(11)]  # Windows re-enabled the listed GPU under a new LUID
        snaps = live.sample(now=live.next_discovery)
        self.assertEqual([s.adapter.luid for s in snaps], [11, tcc.luid])
        self.assertIs(live.slots[1].collector, tcc_collector)
        tcc_collector.close.assert_not_called()
        tcc_collector.nvml.refresh.assert_called_once()  # so TccCounters finds its handle again


if __name__ == "__main__":
    unittest.main()