LLM inference in C/C++
C++ ggml
ggml : shorten virtual device naming in CUDA and Metal (#27608)
Georgi Gerganova14dba686aaa
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex b4c1281419..2456f7dcc6 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -4611,8 +4611,8 @@ static std::string ggml_cuda_device_description(int device) {const ggml_cuda_device_info & info = ggml_cuda_info();std::string description = prop.name;if (info.device_count > info.physical_device_count) {- description += " (physical device " + std::to_string(info.devices[device].physical_device) +- ", virtual device " + std::to_string(info.devices[device].virtual_index) + ")";+ description += " (dev p" + std::to_string(info.devices[device].physical_device) ++ "/v" + std::to_string(info.devices[device].virtual_index) + ")";}return description;}diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex 52043696eb..24caad7ffa 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -17,10 +17,10 @@ struct ggml_metal_device_deleter {typedef std::unique_ptr<ggml_metal_device, ggml_metal_device_deleter> ggml_metal_device_ptr;-ggml_metal_device_t ggml_metal_device_get(int device) {+ggml_metal_device_t ggml_metal_device_get(int device, int n_devices) {static std::vector<ggml_metal_device_ptr> devs;- devs.emplace_back(ggml_metal_device_init(device));+ devs.emplace_back(ggml_metal_device_init(device, n_devices));return devs.back().get();}diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex b7d4660588..5f5410a03a 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -259,6 +259,8 @@ enum ggml_metal_device_id {struct ggml_metal_device_props {int device;+ int device_phys;+ int device_virt;char name[128];char desc[128];@@ -286,10 +288,10 @@ typedef struct ggml_metal_event * ggml_metal_event_t;void ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf);void ggml_metal_event_encode_wait (ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf);-ggml_metal_device_t ggml_metal_device_init(int device);+ggml_metal_device_t ggml_metal_device_init(int device, int n_devices);void ggml_metal_device_free(ggml_metal_device_t dev);-ggml_metal_device_t ggml_metal_device_get(int device);+ggml_metal_device_t ggml_metal_device_get(int device, int n_devices);void * ggml_metal_device_get_obj (ggml_metal_device_t dev); // id<MTLDevice>void * ggml_metal_device_get_queue(ggml_metal_device_t dev); // id<MTLCommandQueue>diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 312b00dc48..263016df7d 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -711,7 +711,7 @@ static enum ggml_metal_device_id ggml_metal_device_id_parse(const char * name) {return GGML_METAL_DEVICE_GENERIC;}-ggml_metal_device_t ggml_metal_device_init(int device) {+ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {ggml_metal_device_t dev = calloc(1, sizeof(struct ggml_metal_device));assert(dev != NULL);@@ -728,6 +728,12 @@ ggml_metal_device_t ggml_metal_device_init(int device) {dev->addr_virt = 0x000000400ULL;dev->props.device = device;++ // the Metal backend uses the system default device as the single physical device;+ // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES+ dev->props.device_phys = 0;+ dev->props.device_virt = device;+dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];@@ -891,7 +897,13 @@ ggml_metal_device_t ggml_metal_device_init(int device) {}snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);- snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", [[dev->mtl_device name] UTF8String]);+ const char * gpu_name = [[dev->mtl_device name] UTF8String];+ if (n_devices > 1) {+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",+ gpu_name, dev->props.device_phys, dev->props.device_virt);+ } else {+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);+ }dev->library = ggml_metal_library_init(dev);if (!dev->library) {diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cppindex 0e8d409e0b..31aa61e32d 100644--- a/ggml/src/ggml-metal/ggml-metal.cpp+++ b/ggml/src/ggml-metal/ggml-metal.cpp@@ -891,7 +891,7 @@ static ggml_backend_dev_t ggml_backend_metal_device_init(ggml_backend_reg_t reg,return new ggml_backend_device {/* .iface = */ ggml_backend_metal_device_i,/* .reg = */ reg,- /* .context = */ ggml_metal_device_get(device),+ /* .context = */ ggml_metal_device_get(device, g_devices),};}