Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
vulkan: Add memory detection for Intel GPU using DXGI+PDH (#12664)
virajwad220e133fca8f
diff --git a/llama/patches/0031-Add-memory-detection-using-DXGI-PDH.patch b/llama/patches/0031-Add-memory-detection-using-DXGI-PDH.patchnew file mode 100644index 000000000..8e5461cbb--- /dev/null+++ b/llama/patches/0031-Add-memory-detection-using-DXGI-PDH.patch@@ -0,0 +1,420 @@+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001+From: Viraj Wadhwa <[email protected]>+Date: Tue, 4 Nov 2025 12:04:04 -0800+Subject: [PATCH] Add memory detection using DXGI + PDH++---+ ggml/src/CMakeLists.txt | 1 ++ ggml/src/ggml-impl.h | 3 ++ ggml/src/ggml-vulkan/ggml-vulkan.cpp | 29 ++-+ ggml/src/mem_dxgi_pdh.cpp | 297 ++++++++++++++++++++++++++++ 4 files changed, 327 insertions(+), 3 deletions(-)+ create mode 100644 ggml/src/mem_dxgi_pdh.cpp++diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt+index aefe43bdd..21fe4640c 100644+--- a/ggml/src/CMakeLists.txt++++ b/ggml/src/CMakeLists.txt+@@ -211,6 +211,7 @@ add_library(ggml-base+ ggml-quants.h+ mem_hip.cpp+ mem_nvml.cpp++ mem_dxgi_pdh.cpp+ gguf.cpp)++ target_include_directories(ggml-base PRIVATE .)+diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h+index b63edd0c1..81cad8cf3 100644+--- a/ggml/src/ggml-impl.h++++ b/ggml/src/ggml-impl.h+@@ -645,6 +645,9 @@ GGML_API void ggml_nvml_release();+ GGML_API int ggml_hip_mgmt_init();+ GGML_API int ggml_hip_get_device_memory(const char *id, size_t *free, size_t *total);+ GGML_API void ggml_hip_mgmt_release();++GGML_API int ggml_dxgi_pdh_init();++GGML_API int ggml_dxgi_pdh_get_device_memory(const char* luid, size_t *free, size_t *total, bool is_integrated_gpu);++GGML_API void ggml_dxgi_pdh_release();++ #ifdef __cplusplus+ }+diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp+index cc68e7968..27d6574da 100644+--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp++++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp+@@ -73,6 +73,7 @@ DispatchLoaderDynamic & ggml_vk_default_dispatcher();+ #define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16"+ #define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000)+ #define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000)++#define VK_LUID_SIZE_KHR VK_LUID_SIZE++ typedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR {+ VkStructureType sType;+@@ -12433,6 +12434,7 @@ struct ggml_backend_vk_device_context {+ std::string pci_id;+ std::string id;+ std::string uuid;++ std::string luid;+ std::string numeric_id;+ int major;+ int minor;+@@ -12449,8 +12451,22 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, size+ vk::PhysicalDeviceMemoryProperties memprops = vkdev.getMemoryProperties();+ vk::PhysicalDeviceProperties2 props2;+ vkdev.getProperties2(&props2);++ GGML_LOG_DEBUG("ggml_backend_vk_get_device_memory called: uuid %s\n", ctx->uuid.c_str());++ GGML_LOG_DEBUG("ggml_backend_vk_get_device_memory called: luid %s\n", ctx->luid.c_str());++++ // Check VRAM reporting for Windows IGPU/DGPU using DXGI + PDH (vendor agnostic)++ if (ggml_dxgi_pdh_init() == 0) {++ GGML_LOG_DEBUG("DXGI + PDH Initialized. Getting GPU free memory info\n");++ int status = ggml_dxgi_pdh_get_device_memory(ctx->luid.c_str(), free, total, ctx->is_integrated_gpu);++ if (status == 0) {++ GGML_LOG_DEBUG("%s utilizing DXGI + PDH memory reporting free: %zu total: %zu\n", __func__, *free, *total);++ ggml_dxgi_pdh_release();++ return;++ }++ ggml_dxgi_pdh_release();++ }++- if (!ctx->is_integrated_gpu)++ if (!ctx->is_integrated_gpu)+ {+ // Use vendor specific management libraries for best VRAM reporting if available+ switch (props2.properties.vendorID) {+@@ -12478,8 +12494,8 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, size+ break;+ }+ }+- // else fallback to memory budget if supported+++ // else fallback to memory budget if supported+ *total = 0;+ *free = 0;+ vk::PhysicalDeviceMemoryBudgetPropertiesEXT mem_budget_props;+@@ -13091,7 +13107,6 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,+ /* .reg = */ reg,+ /* .context = */ ctx,+ });+-+ // Gather additional information about the device+ int dev_idx = vk_instance.device_indices[i];+ vk::PhysicalDeviceProperties props1;+@@ -13114,6 +13129,14 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,+ }+ }+ ctx->uuid = oss.str();++ const auto& luid = device_id_props.deviceLUID;++ char luid_str[32]; // "0x" + 16 hex digits + null terminator = 19 chars++ snprintf(luid_str, sizeof(luid_str), // high part + low part++ "0x%02x%02x%02x%02x%02x%02x%02x%02x",++ luid[7], luid[6], luid[5], luid[4],++ luid[3], luid[2], luid[1], luid[0]++ );++ ctx->luid = std::string(luid_str);+ ctx->major = 0;+ ctx->minor = 0;+ // TODO regex parse driver_props.driverInfo for a X.Y or X.Y.Z version string+diff --git a/ggml/src/mem_dxgi_pdh.cpp b/ggml/src/mem_dxgi_pdh.cpp+new file mode 100644+index 000000000..2f395761c+--- /dev/null++++ b/ggml/src/mem_dxgi_pdh.cpp+@@ -0,0 +1,297 @@++// DXGI and PDH Performance Counters Library++// This Windows-only (10/11) library provides accurate VRAM reporting++#include "ggml.h"++#include "ggml-impl.h"++++#ifdef _WIN32++# define WIN32_LEAN_AND_MEAN++# ifndef NOMINMAX++# define NOMINMAX++# endif++#include <windows.h>++#include <pdh.h>++#include <dxgi1_2.h>++#include <sstream>++#include <thread>++#include <filesystem>++#include <mutex>++++namespace fs = std::filesystem;++++static std::mutex ggml_dxgi_pdh_lock;++++/*++Struct to keep track of GPU adapter information at runtime++*/++struct GpuInfo {++ std::wstring description; // debug field++ LUID luid;++ std::wstring pdhInstance;++ double dedicatedTotal = 0.0;++ double sharedTotal = 0.0;++ double dedicatedUsage = 0.0;++ double sharedUsage = 0.0;++};++++/*++DLL Function Pointers++*/++struct {++ void *dxgi_dll_handle;++ void *pdh_dll_handle;++ // DXGI Functions++ HRESULT (*CreateDXGIFactory1)(REFIID riid, void **ppFactory);++ // PDH functions++ PDH_STATUS (*PdhOpenQueryW)(LPCWSTR szDataSource, DWORD_PTR dwUserData, PDH_HQUERY *phQuery);++ PDH_STATUS (*PdhAddCounterW)(PDH_HQUERY hQuery, LPCWSTR szFullCounterPath, DWORD_PTR dwUserData, PDH_HCOUNTER *phCounter);++ PDH_STATUS (*PdhCollectQueryData)(PDH_HQUERY hQuery);++ PDH_STATUS (*PdhGetFormattedCounterValue)(PDH_HCOUNTER hCounter, DWORD dwFormat, LPDWORD lpdwType, PPDH_FMT_COUNTERVALUE pValue);++ PDH_STATUS (*PdhCloseQuery)(PDH_HQUERY hQuery);++} dll_functions {++ nullptr,nullptr,nullptr,nullptr,nullptr,nullptr,nullptr,nullptr++};++++/*++Create a PDH Instance name++*/++static std::wstring generate_pdh_instance_name_from_luid(const LUID& luid) {++ std::wstringstream ss;++ ss << L"luid_0x" << std::hex << std::setw(8) << std::setfill(L'0') << std::uppercase << luid.HighPart++ << L"_0x" << std::setw(8) << std::setfill(L'0') << luid.LowPart;++ return ss.str();++}++++/*++Conversion from Bytes to GigaBytes++*/++template <typename T>++static inline double b_to_gb(T n)++{++ return (double(n) / (1024.0 * 1024 * 1024));++}++++/*++Fetch the GPU adapter 'dedicated memory' and 'shared memory' using DXGI++*/++static void fetch_dxgi_adapter_desc1(const DXGI_ADAPTER_DESC1& desc, GpuInfo* info) {++ auto dedicatedVideoMemory = desc.DedicatedVideoMemory;++ auto sharedSystemMemory = desc.SharedSystemMemory;++ GGML_LOG_DEBUG("[DXGI] Adapter Description: %ls, LUID: 0x%08X%08X, Dedicated: %.2f GB, Shared: %.2f GB\n", desc.Description, desc.AdapterLuid.HighPart, desc.AdapterLuid.LowPart, b_to_gb(dedicatedVideoMemory), b_to_gb(sharedSystemMemory));++ if (info) {++ info->dedicatedTotal = dedicatedVideoMemory; // values in bytes++ info->sharedTotal = sharedSystemMemory;++ }++}++++/*++Enumerate over the GPU adapters detected using DXGI and return their information++*/++static std::vector<GpuInfo> get_dxgi_gpu_infos() {++ std::vector<GpuInfo> infos;++ IDXGIFactory1* pFactory = nullptr;++++ if (SUCCEEDED(dll_functions.CreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)&pFactory))) {++ UINT i = 0;++ IDXGIAdapter1* pAdapter = nullptr;++ while (pFactory->EnumAdapters1(i, &pAdapter) != DXGI_ERROR_NOT_FOUND) {++ DXGI_ADAPTER_DESC1 desc;++ pAdapter->GetDesc1(&desc);++++ // Get all the GPU adapter info++ GpuInfo info;++ fetch_dxgi_adapter_desc1(desc, &info);++ info.description = std::wstring(desc.Description);++ info.luid = desc.AdapterLuid;++ info.pdhInstance = generate_pdh_instance_name_from_luid(desc.AdapterLuid);++ infos.push_back(info);++++ pAdapter->Release();++ ++i;++ }++ pFactory->Release();++ }++ return infos;++}++++static bool get_gpu_memory_usage(GpuInfo& gpu) {++ PDH_HQUERY query;++ if (dll_functions.PdhOpenQueryW(NULL, 0, &query) != ERROR_SUCCESS) {++ return false;++ }++++ struct GpuCounters {++ PDH_HCOUNTER dedicated;++ PDH_HCOUNTER shared;++ };++++ GpuCounters gpuCounter{};++++ std::wstring dedicatedPath = L"\\GPU Adapter Memory(" + gpu.pdhInstance + L"*)\\Dedicated Usage";++ std::wstring sharedPath = L"\\GPU Adapter Memory(" + gpu.pdhInstance + L"*)\\Shared Usage";++++ if (dll_functions.PdhAddCounterW(query, dedicatedPath.c_str(), 0, &gpuCounter.dedicated) != ERROR_SUCCESS ||++ dll_functions.PdhAddCounterW(query, sharedPath.c_str(), 0, &gpuCounter.shared) != ERROR_SUCCESS) {++ GGML_LOG_ERROR("Failed to add PDH counters for GPU %s\n", std::string(gpu.pdhInstance.begin(), gpu.pdhInstance.end()).c_str());++ dll_functions.PdhCloseQuery(query);++ return false;++ }++++ // Sample the data++ if (dll_functions.PdhCollectQueryData(query) != ERROR_SUCCESS) {++ dll_functions.PdhCloseQuery(query);++ return false;++ }++++ // Read final values++ PDH_FMT_COUNTERVALUE val;++++ if (dll_functions.PdhGetFormattedCounterValue(gpuCounter.dedicated, PDH_FMT_DOUBLE, NULL, &val) == ERROR_SUCCESS)++ gpu.dedicatedUsage = val.doubleValue;++++ if (dll_functions.PdhGetFormattedCounterValue(gpuCounter.shared, PDH_FMT_DOUBLE, NULL, &val) == ERROR_SUCCESS)++ gpu.sharedUsage = val.doubleValue;++++ dll_functions.PdhCloseQuery(query);++ return true;++}++++++extern "C" {++++ int ggml_dxgi_pdh_init() {++ GGML_LOG_DEBUG("%s called\n", __func__);++ std::lock_guard<std::mutex> lock(ggml_dxgi_pdh_lock);++ if (dll_functions.dxgi_dll_handle != NULL && dll_functions.pdh_dll_handle != NULL) {++ // Already initialized as we have both DLL handles++ return ERROR_SUCCESS;++ }++++ DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);++ SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);++ fs::path libPath_dxgi = fs::path("\\Windows") / fs::path("System32") / fs::path("dxgi.dll");++ fs::path libPath_pdh = fs::path("\\Windows") / fs::path("System32") / fs::path("pdh.dll");++++ // Call LoadLibraryW on both DLLs to ensure they are loaded++ void *dxgi = (void*)LoadLibraryW(libPath_dxgi.wstring().c_str());++ void *pdh = (void*)LoadLibraryW(libPath_pdh.wstring().c_str());++ if(dxgi == NULL || pdh == NULL) {++ if (dxgi != NULL) {++ FreeLibrary((HMODULE)(dxgi));++ }++ if (pdh != NULL) {++ FreeLibrary((HMODULE)(pdh));++ }++ SetErrorMode(old_mode);++ return ERROR_DLL_NOT_FOUND;++ }++ else {++ // save the dll handles++ dll_functions.dxgi_dll_handle = dxgi;++ dll_functions.pdh_dll_handle = pdh;++ }++++ // Get pointers to the library functions loaded by the DLLs++ dll_functions.CreateDXGIFactory1 = (HRESULT (*)(REFIID riid, void **ppFactory)) GetProcAddress((HMODULE)(dll_functions.dxgi_dll_handle), "CreateDXGIFactory1");++ dll_functions.PdhOpenQueryW = (PDH_STATUS (*)(LPCWSTR szDataSource, DWORD_PTR dwUserData, PDH_HQUERY *phQuery)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhOpenQueryW");++ dll_functions.PdhAddCounterW = (PDH_STATUS (*)(PDH_HQUERY hQuery, LPCWSTR szFullCounterPath, DWORD_PTR dwUserData, PDH_HCOUNTER *phCounter)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhAddCounterW");++ dll_functions.PdhCollectQueryData = (PDH_STATUS (*)(PDH_HQUERY hQuery)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhCollectQueryData");++ dll_functions.PdhGetFormattedCounterValue = (PDH_STATUS (*)(PDH_HCOUNTER hCounter, DWORD dwFormat, LPDWORD lpdwType, PPDH_FMT_COUNTERVALUE pValue)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhGetFormattedCounterValue");++ dll_functions.PdhCloseQuery = (PDH_STATUS (*)(PDH_HQUERY hQuery)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhCloseQuery");++++ SetErrorMode(old_mode); // set old mode before any return++++ // Check if any function pointers are NULL (not found)++ if (dll_functions.CreateDXGIFactory1 == NULL || dll_functions.PdhOpenQueryW == NULL || dll_functions.PdhAddCounterW == NULL || dll_functions.PdhCollectQueryData == NULL || dll_functions.PdhGetFormattedCounterValue == NULL || dll_functions.PdhCloseQuery == NULL) {++ GGML_LOG_INFO("%s unable to locate required symbols in either dxgi.dll or pdh.dll", __func__);++ FreeLibrary((HMODULE)(dll_functions.dxgi_dll_handle));++ FreeLibrary((HMODULE)(dll_functions.pdh_dll_handle));++ dll_functions.dxgi_dll_handle = NULL;++ dll_functions.pdh_dll_handle = NULL;++ return ERROR_PROC_NOT_FOUND;++ }++++ // No other initializations needed, successfully loaded the libraries and functions!++ return ERROR_SUCCESS;++ }++++ void ggml_dxgi_pdh_release() {++ std::lock_guard<std::mutex> lock(ggml_dxgi_pdh_lock);++ if (dll_functions.dxgi_dll_handle == NULL && dll_functions.pdh_dll_handle == NULL) {++ // Already freed the DLLs++ return;++ }++++ // Call FreeLibrary on both DLLs++ FreeLibrary((HMODULE)(dll_functions.dxgi_dll_handle));++ FreeLibrary((HMODULE)(dll_functions.pdh_dll_handle));++++ dll_functions.dxgi_dll_handle = NULL;++ dll_functions.pdh_dll_handle = NULL;++++ return; // successfully released++ }++++ int ggml_dxgi_pdh_get_device_memory(const char* luid, size_t *free, size_t *total, bool is_integrated_gpu) {++++ std::lock_guard<std::mutex> lock(ggml_dxgi_pdh_lock);++++ // Enumerate GPUs using DXGI and find the matching LUID++ // This also fetches the total memory info for each of the enumerated GPUs++ std::vector<GpuInfo> gpus = get_dxgi_gpu_infos();++ GpuInfo *targetGpu = nullptr;++ for (auto& gpu : gpus) {++ char luid_buffer[32]; // "0x" + 16 hex digits + null terminator++ snprintf(luid_buffer, sizeof(luid_buffer), "0x%08x%08x", gpu.luid.HighPart, gpu.luid.LowPart);++ std::string gpu_luid_str(luid_buffer);++ if (gpu_luid_str == std::string(luid)) {++ targetGpu = &gpu;++ break;++ }++ }++ if (!targetGpu) {++ GGML_LOG_ERROR("GPU with LUID %s not found.\n", luid);++ return ERROR_NOT_FOUND;++ }++++ // Get the current memory usage for the target GPU++ int status = get_gpu_memory_usage(*targetGpu);++ if (!status) {++ GGML_LOG_ERROR("Failed to get GPU memory usage.\n");++ return ERROR_DEVICE_NOT_AVAILABLE;++ }++++ // Calculate the free memory based on whether it's an integrated or discrete GPU++ if (is_integrated_gpu) {++ // IGPU free = SharedTotal - SharedUsage++ GGML_LOG_DEBUG("Integrated GPU (%ls) with LUID %s detected. Shared Total: %.2f bytes (%.2f GB), Shared Usage: %.2f bytes (%.2f GB), Dedicated Total: %.2f bytes (%.2f GB), Dedicated Usage: %.2f bytes (%.2f GB)\n", targetGpu->description.c_str(), luid, targetGpu->sharedTotal, b_to_gb(targetGpu->sharedTotal), targetGpu->sharedUsage, b_to_gb(targetGpu->sharedUsage), targetGpu->dedicatedTotal, b_to_gb(targetGpu->dedicatedTotal), targetGpu->dedicatedUsage, b_to_gb(targetGpu->dedicatedUsage));++ *free = (targetGpu->sharedTotal - targetGpu->sharedUsage) + (targetGpu->dedicatedTotal - targetGpu->dedicatedUsage); // Some IGPUs also have dedicated memory, which can be used along with the IGPU's shared memory++ *total = targetGpu->sharedTotal + targetGpu->dedicatedTotal;++ }++ else {++ // DGPU free = DedicatedTotal - DedicatedUsage++ GGML_LOG_DEBUG("Discrete GPU (%ls) with LUID %s detected. Dedicated Total: %.2f bytes (%.2f GB), Dedicated Usage: %.2f bytes (%.2f GB)\n", targetGpu->description.c_str(), luid, targetGpu->dedicatedTotal, b_to_gb(targetGpu->dedicatedTotal), targetGpu->dedicatedUsage, b_to_gb(targetGpu->dedicatedUsage));++ *free = targetGpu->dedicatedTotal - targetGpu->dedicatedUsage;++ *total = targetGpu->dedicatedTotal;++ }++++ return ERROR_SUCCESS;++ }++++} // extern "C"++++#else // #ifdef _WIN32++++extern "C" {++++ // DXGI + PDH not available for Linux implementation++ int ggml_dxgi_pdh_init() {++ return -1;++ }++ void ggml_dxgi_pdh_release() {}++ int ggml_dxgi_pdh_get_device_memory(const char* luid, size_t *free, size_t *total, bool is_integrated_gpu) {++ return -1;++ }++++} // extern "C"++++#endif // #ifdef _WIN32+\ No newline at end of filediff --git a/ml/backend/ggml/ggml/src/CMakeLists.txt b/ml/backend/ggml/ggml/src/CMakeLists.txtindex aefe43bdd..21fe4640c 100644--- a/ml/backend/ggml/ggml/src/CMakeLists.txt+++ b/ml/backend/ggml/ggml/src/CMakeLists.txt@@ -211,6 +211,7 @@ add_library(ggml-baseggml-quants.hmem_hip.cppmem_nvml.cpp+ mem_dxgi_pdh.cppgguf.cpp)target_include_directories(ggml-base PRIVATE .)diff --git a/ml/backend/ggml/ggml/src/ggml-impl.h b/ml/backend/ggml/ggml/src/ggml-impl.hindex b63edd0c1..81cad8cf3 100644--- a/ml/backend/ggml/ggml/src/ggml-impl.h+++ b/ml/backend/ggml/ggml/src/ggml-impl.h@@ -645,6 +645,9 @@ GGML_API void ggml_nvml_release();GGML_API int ggml_hip_mgmt_init();GGML_API int ggml_hip_get_device_memory(const char *id, size_t *free, size_t *total);GGML_API void ggml_hip_mgmt_release();+GGML_API int ggml_dxgi_pdh_init();+GGML_API int ggml_dxgi_pdh_get_device_memory(const char* luid, size_t *free, size_t *total, bool is_integrated_gpu);+GGML_API void ggml_dxgi_pdh_release();#ifdef __cplusplus}diff --git a/ml/backend/ggml/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ml/backend/ggml/ggml/src/ggml-vulkan/ggml-vulkan.cppindex cc68e7968..27d6574da 100644--- a/ml/backend/ggml/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ml/backend/ggml/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -73,6 +73,7 @@ DispatchLoaderDynamic & ggml_vk_default_dispatcher();#define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16"#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000)#define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000)+#define VK_LUID_SIZE_KHR VK_LUID_SIZEtypedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR {VkStructureType sType;@@ -12433,6 +12434,7 @@ struct ggml_backend_vk_device_context {std::string pci_id;std::string id;std::string uuid;+ std::string luid;std::string numeric_id;int major;int minor;@@ -12449,8 +12451,22 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, sizevk::PhysicalDeviceMemoryProperties memprops = vkdev.getMemoryProperties();vk::PhysicalDeviceProperties2 props2;vkdev.getProperties2(&props2);+ GGML_LOG_DEBUG("ggml_backend_vk_get_device_memory called: uuid %s\n", ctx->uuid.c_str());+ GGML_LOG_DEBUG("ggml_backend_vk_get_device_memory called: luid %s\n", ctx->luid.c_str());++ // Check VRAM reporting for Windows IGPU/DGPU using DXGI + PDH (vendor agnostic)+ if (ggml_dxgi_pdh_init() == 0) {+ GGML_LOG_DEBUG("DXGI + PDH Initialized. Getting GPU free memory info\n");+ int status = ggml_dxgi_pdh_get_device_memory(ctx->luid.c_str(), free, total, ctx->is_integrated_gpu);+ if (status == 0) {+ GGML_LOG_DEBUG("%s utilizing DXGI + PDH memory reporting free: %zu total: %zu\n", __func__, *free, *total);+ ggml_dxgi_pdh_release();+ return;+ }+ ggml_dxgi_pdh_release();+ }- if (!ctx->is_integrated_gpu)+ if (!ctx->is_integrated_gpu){// Use vendor specific management libraries for best VRAM reporting if availableswitch (props2.properties.vendorID) {@@ -12478,8 +12494,8 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, sizebreak;}}- // else fallback to memory budget if supported+ // else fallback to memory budget if supported*total = 0;*free = 0;vk::PhysicalDeviceMemoryBudgetPropertiesEXT mem_budget_props;@@ -13091,7 +13107,6 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,/* .reg = */ reg,/* .context = */ ctx,});-// Gather additional information about the deviceint dev_idx = vk_instance.device_indices[i];vk::PhysicalDeviceProperties props1;@@ -13114,6 +13129,14 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,}}ctx->uuid = oss.str();+ const auto& luid = device_id_props.deviceLUID;+ char luid_str[32]; // "0x" + 16 hex digits + null terminator = 19 chars+ snprintf(luid_str, sizeof(luid_str), // high part + low part+ "0x%02x%02x%02x%02x%02x%02x%02x%02x",+ luid[7], luid[6], luid[5], luid[4],+ luid[3], luid[2], luid[1], luid[0]+ );+ ctx->luid = std::string(luid_str);ctx->major = 0;ctx->minor = 0;// TODO regex parse driver_props.driverInfo for a X.Y or X.Y.Z version stringdiff --git a/ml/backend/ggml/ggml/src/mem_dxgi_pdh.cpp b/ml/backend/ggml/ggml/src/mem_dxgi_pdh.cppnew file mode 100644index 000000000..2f395761c--- /dev/null+++ b/ml/backend/ggml/ggml/src/mem_dxgi_pdh.cpp@@ -0,0 +1,297 @@+// DXGI and PDH Performance Counters Library+// This Windows-only (10/11) library provides accurate VRAM reporting+#include "ggml.h"+#include "ggml-impl.h"++#ifdef _WIN32+# define WIN32_LEAN_AND_MEAN+# ifndef NOMINMAX+# define NOMINMAX+# endif+#include <windows.h>+#include <pdh.h>+#include <dxgi1_2.h>+#include <sstream>+#include <thread>+#include <filesystem>+#include <mutex>++namespace fs = std::filesystem;++static std::mutex ggml_dxgi_pdh_lock;++/*+Struct to keep track of GPU adapter information at runtime+*/+struct GpuInfo {+ std::wstring description; // debug field+ LUID luid;+ std::wstring pdhInstance;+ double dedicatedTotal = 0.0;+ double sharedTotal = 0.0;+ double dedicatedUsage = 0.0;+ double sharedUsage = 0.0;+};++/*+DLL Function Pointers+*/+struct {+ void *dxgi_dll_handle;+ void *pdh_dll_handle;+ // DXGI Functions+ HRESULT (*CreateDXGIFactory1)(REFIID riid, void **ppFactory);+ // PDH functions+ PDH_STATUS (*PdhOpenQueryW)(LPCWSTR szDataSource, DWORD_PTR dwUserData, PDH_HQUERY *phQuery);+ PDH_STATUS (*PdhAddCounterW)(PDH_HQUERY hQuery, LPCWSTR szFullCounterPath, DWORD_PTR dwUserData, PDH_HCOUNTER *phCounter);+ PDH_STATUS (*PdhCollectQueryData)(PDH_HQUERY hQuery);+ PDH_STATUS (*PdhGetFormattedCounterValue)(PDH_HCOUNTER hCounter, DWORD dwFormat, LPDWORD lpdwType, PPDH_FMT_COUNTERVALUE pValue);+ PDH_STATUS (*PdhCloseQuery)(PDH_HQUERY hQuery);+} dll_functions {+ nullptr,nullptr,nullptr,nullptr,nullptr,nullptr,nullptr,nullptr+};++/*+Create a PDH Instance name+*/+static std::wstring generate_pdh_instance_name_from_luid(const LUID& luid) {+ std::wstringstream ss;+ ss << L"luid_0x" << std::hex << std::setw(8) << std::setfill(L'0') << std::uppercase << luid.HighPart+ << L"_0x" << std::setw(8) << std::setfill(L'0') << luid.LowPart;+ return ss.str();+}++/*+Conversion from Bytes to GigaBytes+*/+template <typename T>+static inline double b_to_gb(T n)+{+ return (double(n) / (1024.0 * 1024 * 1024));+}++/*+Fetch the GPU adapter 'dedicated memory' and 'shared memory' using DXGI+*/+static void fetch_dxgi_adapter_desc1(const DXGI_ADAPTER_DESC1& desc, GpuInfo* info) {+ auto dedicatedVideoMemory = desc.DedicatedVideoMemory;+ auto sharedSystemMemory = desc.SharedSystemMemory;+ GGML_LOG_DEBUG("[DXGI] Adapter Description: %ls, LUID: 0x%08X%08X, Dedicated: %.2f GB, Shared: %.2f GB\n", desc.Description, desc.AdapterLuid.HighPart, desc.AdapterLuid.LowPart, b_to_gb(dedicatedVideoMemory), b_to_gb(sharedSystemMemory));+ if (info) {+ info->dedicatedTotal = dedicatedVideoMemory; // values in bytes+ info->sharedTotal = sharedSystemMemory;+ }+}++/*+Enumerate over the GPU adapters detected using DXGI and return their information+*/+static std::vector<GpuInfo> get_dxgi_gpu_infos() {+ std::vector<GpuInfo> infos;+ IDXGIFactory1* pFactory = nullptr;++ if (SUCCEEDED(dll_functions.CreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)&pFactory))) {+ UINT i = 0;+ IDXGIAdapter1* pAdapter = nullptr;+ while (pFactory->EnumAdapters1(i, &pAdapter) != DXGI_ERROR_NOT_FOUND) {+ DXGI_ADAPTER_DESC1 desc;+ pAdapter->GetDesc1(&desc);++ // Get all the GPU adapter info+ GpuInfo info;+ fetch_dxgi_adapter_desc1(desc, &info);+ info.description = std::wstring(desc.Description);+ info.luid = desc.AdapterLuid;+ info.pdhInstance = generate_pdh_instance_name_from_luid(desc.AdapterLuid);+ infos.push_back(info);++ pAdapter->Release();+ ++i;+ }+ pFactory->Release();+ }+ return infos;+}++static bool get_gpu_memory_usage(GpuInfo& gpu) {+ PDH_HQUERY query;+ if (dll_functions.PdhOpenQueryW(NULL, 0, &query) != ERROR_SUCCESS) {+ return false;+ }++ struct GpuCounters {+ PDH_HCOUNTER dedicated;+ PDH_HCOUNTER shared;+ };++ GpuCounters gpuCounter{};++ std::wstring dedicatedPath = L"\\GPU Adapter Memory(" + gpu.pdhInstance + L"*)\\Dedicated Usage";+ std::wstring sharedPath = L"\\GPU Adapter Memory(" + gpu.pdhInstance + L"*)\\Shared Usage";++ if (dll_functions.PdhAddCounterW(query, dedicatedPath.c_str(), 0, &gpuCounter.dedicated) != ERROR_SUCCESS ||+ dll_functions.PdhAddCounterW(query, sharedPath.c_str(), 0, &gpuCounter.shared) != ERROR_SUCCESS) {+ GGML_LOG_ERROR("Failed to add PDH counters for GPU %s\n", std::string(gpu.pdhInstance.begin(), gpu.pdhInstance.end()).c_str());+ dll_functions.PdhCloseQuery(query);+ return false;+ }++ // Sample the data+ if (dll_functions.PdhCollectQueryData(query) != ERROR_SUCCESS) {+ dll_functions.PdhCloseQuery(query);+ return false;+ }++ // Read final values+ PDH_FMT_COUNTERVALUE val;++ if (dll_functions.PdhGetFormattedCounterValue(gpuCounter.dedicated, PDH_FMT_DOUBLE, NULL, &val) == ERROR_SUCCESS)+ gpu.dedicatedUsage = val.doubleValue;++ if (dll_functions.PdhGetFormattedCounterValue(gpuCounter.shared, PDH_FMT_DOUBLE, NULL, &val) == ERROR_SUCCESS)+ gpu.sharedUsage = val.doubleValue;++ dll_functions.PdhCloseQuery(query);+ return true;+}+++extern "C" {++ int ggml_dxgi_pdh_init() {+ GGML_LOG_DEBUG("%s called\n", __func__);+ std::lock_guard<std::mutex> lock(ggml_dxgi_pdh_lock);+ if (dll_functions.dxgi_dll_handle != NULL && dll_functions.pdh_dll_handle != NULL) {+ // Already initialized as we have both DLL handles+ return ERROR_SUCCESS;+ }++ DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);+ SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);+ fs::path libPath_dxgi = fs::path("\\Windows") / fs::path("System32") / fs::path("dxgi.dll");+ fs::path libPath_pdh = fs::path("\\Windows") / fs::path("System32") / fs::path("pdh.dll");++ // Call LoadLibraryW on both DLLs to ensure they are loaded+ void *dxgi = (void*)LoadLibraryW(libPath_dxgi.wstring().c_str());+ void *pdh = (void*)LoadLibraryW(libPath_pdh.wstring().c_str());+ if(dxgi == NULL || pdh == NULL) {+ if (dxgi != NULL) {+ FreeLibrary((HMODULE)(dxgi));+ }+ if (pdh != NULL) {+ FreeLibrary((HMODULE)(pdh));+ }+ SetErrorMode(old_mode);+ return ERROR_DLL_NOT_FOUND;+ }+ else {+ // save the dll handles+ dll_functions.dxgi_dll_handle = dxgi;+ dll_functions.pdh_dll_handle = pdh;+ }++ // Get pointers to the library functions loaded by the DLLs+ dll_functions.CreateDXGIFactory1 = (HRESULT (*)(REFIID riid, void **ppFactory)) GetProcAddress((HMODULE)(dll_functions.dxgi_dll_handle), "CreateDXGIFactory1");+ dll_functions.PdhOpenQueryW = (PDH_STATUS (*)(LPCWSTR szDataSource, DWORD_PTR dwUserData, PDH_HQUERY *phQuery)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhOpenQueryW");+ dll_functions.PdhAddCounterW = (PDH_STATUS (*)(PDH_HQUERY hQuery, LPCWSTR szFullCounterPath, DWORD_PTR dwUserData, PDH_HCOUNTER *phCounter)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhAddCounterW");+ dll_functions.PdhCollectQueryData = (PDH_STATUS (*)(PDH_HQUERY hQuery)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhCollectQueryData");+ dll_functions.PdhGetFormattedCounterValue = (PDH_STATUS (*)(PDH_HCOUNTER hCounter, DWORD dwFormat, LPDWORD lpdwType, PPDH_FMT_COUNTERVALUE pValue)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhGetFormattedCounterValue");+ dll_functions.PdhCloseQuery = (PDH_STATUS (*)(PDH_HQUERY hQuery)) GetProcAddress((HMODULE)(dll_functions.pdh_dll_handle), "PdhCloseQuery");++ SetErrorMode(old_mode); // set old mode before any return++ // Check if any function pointers are NULL (not found)+ if (dll_functions.CreateDXGIFactory1 == NULL || dll_functions.PdhOpenQueryW == NULL || dll_functions.PdhAddCounterW == NULL || dll_functions.PdhCollectQueryData == NULL || dll_functions.PdhGetFormattedCounterValue == NULL || dll_functions.PdhCloseQuery == NULL) {+ GGML_LOG_INFO("%s unable to locate required symbols in either dxgi.dll or pdh.dll", __func__);+ FreeLibrary((HMODULE)(dll_functions.dxgi_dll_handle));+ FreeLibrary((HMODULE)(dll_functions.pdh_dll_handle));+ dll_functions.dxgi_dll_handle = NULL;+ dll_functions.pdh_dll_handle = NULL;+ return ERROR_PROC_NOT_FOUND;+ }++ // No other initializations needed, successfully loaded the libraries and functions!+ return ERROR_SUCCESS;+ }++ void ggml_dxgi_pdh_release() {+ std::lock_guard<std::mutex> lock(ggml_dxgi_pdh_lock);+ if (dll_functions.dxgi_dll_handle == NULL && dll_functions.pdh_dll_handle == NULL) {+ // Already freed the DLLs+ return;+ }++ // Call FreeLibrary on both DLLs+ FreeLibrary((HMODULE)(dll_functions.dxgi_dll_handle));+ FreeLibrary((HMODULE)(dll_functions.pdh_dll_handle));++ dll_functions.dxgi_dll_handle = NULL;+ dll_functions.pdh_dll_handle = NULL;++ return; // successfully released+ }++ int ggml_dxgi_pdh_get_device_memory(const char* luid, size_t *free, size_t *total, bool is_integrated_gpu) {++ std::lock_guard<std::mutex> lock(ggml_dxgi_pdh_lock);++ // Enumerate GPUs using DXGI and find the matching LUID+ // This also fetches the total memory info for each of the enumerated GPUs+ std::vector<GpuInfo> gpus = get_dxgi_gpu_infos();+ GpuInfo *targetGpu = nullptr;+ for (auto& gpu : gpus) {+ char luid_buffer[32]; // "0x" + 16 hex digits + null terminator+ snprintf(luid_buffer, sizeof(luid_buffer), "0x%08x%08x", gpu.luid.HighPart, gpu.luid.LowPart);+ std::string gpu_luid_str(luid_buffer);+ if (gpu_luid_str == std::string(luid)) {+ targetGpu = &gpu;+ break;+ }+ }+ if (!targetGpu) {+ GGML_LOG_ERROR("GPU with LUID %s not found.\n", luid);+ return ERROR_NOT_FOUND;+ }++ // Get the current memory usage for the target GPU+ int status = get_gpu_memory_usage(*targetGpu);+ if (!status) {+ GGML_LOG_ERROR("Failed to get GPU memory usage.\n");+ return ERROR_DEVICE_NOT_AVAILABLE;+ }++ // Calculate the free memory based on whether it's an integrated or discrete GPU+ if (is_integrated_gpu) {+ // IGPU free = SharedTotal - SharedUsage+ GGML_LOG_DEBUG("Integrated GPU (%ls) with LUID %s detected. Shared Total: %.2f bytes (%.2f GB), Shared Usage: %.2f bytes (%.2f GB), Dedicated Total: %.2f bytes (%.2f GB), Dedicated Usage: %.2f bytes (%.2f GB)\n", targetGpu->description.c_str(), luid, targetGpu->sharedTotal, b_to_gb(targetGpu->sharedTotal), targetGpu->sharedUsage, b_to_gb(targetGpu->sharedUsage), targetGpu->dedicatedTotal, b_to_gb(targetGpu->dedicatedTotal), targetGpu->dedicatedUsage, b_to_gb(targetGpu->dedicatedUsage));+ *free = (targetGpu->sharedTotal - targetGpu->sharedUsage) + (targetGpu->dedicatedTotal - targetGpu->dedicatedUsage); // Some IGPUs also have dedicated memory, which can be used along with the IGPU's shared memory+ *total = targetGpu->sharedTotal + targetGpu->dedicatedTotal;+ }+ else {+ // DGPU free = DedicatedTotal - DedicatedUsage+ GGML_LOG_DEBUG("Discrete GPU (%ls) with LUID %s detected. Dedicated Total: %.2f bytes (%.2f GB), Dedicated Usage: %.2f bytes (%.2f GB)\n", targetGpu->description.c_str(), luid, targetGpu->dedicatedTotal, b_to_gb(targetGpu->dedicatedTotal), targetGpu->dedicatedUsage, b_to_gb(targetGpu->dedicatedUsage));+ *free = targetGpu->dedicatedTotal - targetGpu->dedicatedUsage;+ *total = targetGpu->dedicatedTotal;+ }++ return ERROR_SUCCESS;+ }++} // extern "C"++#else // #ifdef _WIN32++extern "C" {++ // DXGI + PDH not available for Linux implementation+ int ggml_dxgi_pdh_init() {+ return -1;+ }+ void ggml_dxgi_pdh_release() {}+ int ggml_dxgi_pdh_get_device_memory(const char* luid, size_t *free, size_t *total, bool is_integrated_gpu) {+ return -1;+ }++} // extern "C"++#endif // #ifdef _WIN32\ No newline at end of file