Skip to content

LLM inference in C/C++

C++ ggml

ggml : shorten virtual device naming in CUDA and Metal (#27608)

Georgi Gerganova14dba686aaa
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index b4c1281419..2456f7dcc6 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -4611,8 +4611,8 @@ static std::string ggml_cuda_device_description(int device) {
const ggml_cuda_device_info & info = ggml_cuda_info();
std::string description = prop.name;
if (info.device_count > info.physical_device_count) {
- description += " (physical device " + std::to_string(info.devices[device].physical_device) +
- ", virtual device " + std::to_string(info.devices[device].virtual_index) + ")";
+ description += " (dev p" + std::to_string(info.devices[device].physical_device) +
+ "/v" + std::to_string(info.devices[device].virtual_index) + ")";
}
return description;
}
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index 52043696eb..24caad7ffa 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -17,10 +17,10 @@ struct ggml_metal_device_deleter {
typedef std::unique_ptr<ggml_metal_device, ggml_metal_device_deleter> ggml_metal_device_ptr;
-ggml_metal_device_t ggml_metal_device_get(int device) {
+ggml_metal_device_t ggml_metal_device_get(int device, int n_devices) {
static std::vector<ggml_metal_device_ptr> devs;
- devs.emplace_back(ggml_metal_device_init(device));
+ devs.emplace_back(ggml_metal_device_init(device, n_devices));
return devs.back().get();
}
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index b7d4660588..5f5410a03a 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -259,6 +259,8 @@ enum ggml_metal_device_id {
struct ggml_metal_device_props {
int device;
+ int device_phys;
+ int device_virt;
char name[128];
char desc[128];
@@ -286,10 +288,10 @@ typedef struct ggml_metal_event * ggml_metal_event_t;
void ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf);
void ggml_metal_event_encode_wait (ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf);
-ggml_metal_device_t ggml_metal_device_init(int device);
+ggml_metal_device_t ggml_metal_device_init(int device, int n_devices);
void ggml_metal_device_free(ggml_metal_device_t dev);
-ggml_metal_device_t ggml_metal_device_get(int device);
+ggml_metal_device_t ggml_metal_device_get(int device, int n_devices);
void * ggml_metal_device_get_obj (ggml_metal_device_t dev); // id<MTLDevice>
void * ggml_metal_device_get_queue(ggml_metal_device_t dev); // id<MTLCommandQueue>
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 312b00dc48..263016df7d 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -711,7 +711,7 @@ static enum ggml_metal_device_id ggml_metal_device_id_parse(const char * name) {
return GGML_METAL_DEVICE_GENERIC;
}
-ggml_metal_device_t ggml_metal_device_init(int device) {
+ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
ggml_metal_device_t dev = calloc(1, sizeof(struct ggml_metal_device));
assert(dev != NULL);
@@ -728,6 +728,12 @@ ggml_metal_device_t ggml_metal_device_init(int device) {
dev->addr_virt = 0x000000400ULL;
dev->props.device = device;
+
+ // the Metal backend uses the system default device as the single physical device;
+ // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES
+ dev->props.device_phys = 0;
+ dev->props.device_virt = device;
+
dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
@@ -891,7 +897,13 @@ ggml_metal_device_t ggml_metal_device_init(int device) {
}
snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
- snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", [[dev->mtl_device name] UTF8String]);
+ const char * gpu_name = [[dev->mtl_device name] UTF8String];
+ if (n_devices > 1) {
+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",
+ gpu_name, dev->props.device_phys, dev->props.device_virt);
+ } else {
+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);
+ }
dev->library = ggml_metal_library_init(dev);
if (!dev->library) {
diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp
index 0e8d409e0b..31aa61e32d 100644
--- a/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/ggml/src/ggml-metal/ggml-metal.cpp
@@ -891,7 +891,7 @@ static ggml_backend_dev_t ggml_backend_metal_device_init(ggml_backend_reg_t reg,
return new ggml_backend_device {
/* .iface = */ ggml_backend_metal_device_i,
/* .reg = */ reg,
- /* .context = */ ggml_metal_device_get(device),
+ /* .context = */ ggml_metal_device_get(device, g_devices),
};
}