Skip to content

LLM inference in C/C++

C++ ggml

metal : fix memory leaks due to missing autoreleasepools (#27758)

Niklas Wenzel915dc6d38c9b
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m
index 32d97cd5d0..1227ed39a0 100644
--- a/ggml/src/ggml-metal/ggml-metal-context.m
+++ b/ggml/src/ggml-metal/ggml-metal-context.m
@@ -84,106 +84,108 @@ struct ggml_metal {
ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
GGML_LOG_INFO("%s: allocating\n", __func__);
+ @autoreleasepool {
#if TARGET_OS_OSX && !GGML_METAL_NDEBUG
- // Show all the Metal device instances in the system
- NSArray * devices = MTLCopyAllDevices();
- for (id<MTLDevice> device in devices) {
- GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);
- }
- [devices release]; // since it was created by a *Copy* C method
+ // Show all the Metal device instances in the system
+ NSArray * devices = MTLCopyAllDevices();
+ for (id<MTLDevice> device in devices) {
+ GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);
+ }
+ [devices release]; // since it was created by a *Copy* C method
#endif
- // init context
- ggml_metal_t res = calloc(1, sizeof(struct ggml_metal));
-
- id<MTLDevice> device = ggml_metal_device_get_obj(dev);
+ // init context
+ ggml_metal_t res = calloc(1, sizeof(struct ggml_metal));
- GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);
+ id<MTLDevice> device = ggml_metal_device_get_obj(dev);
- // TODO: would it be better to have one queue for the backend and one queue for the device?
- // the graph encoders and async ops would use the backend queue while the sync ops would use the device queue?
- //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND]
- id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev);
- if (queue == nil) {
- GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
- return NULL;
- }
+ GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);
- res->dev = dev;
- res->lib = ggml_metal_device_get_library(dev);
- if (res->lib == NULL) {
- GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__);
- GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__);
+ // TODO: would it be better to have one queue for the backend and one queue for the device?
+ // the graph encoders and async ops would use the backend queue while the sync ops would use the device queue?
+ //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND]
+ id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev);
+ if (queue == nil) {
+ GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
+ return NULL;
+ }
- res->lib = ggml_metal_library_init(dev);
+ res->dev = dev;
+ res->lib = ggml_metal_device_get_library(dev);
if (res->lib == NULL) {
- GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__);
+ GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__);
+ GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__);
- free(res);
+ res->lib = ggml_metal_library_init(dev);
+ if (res->lib == NULL) {
+ GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__);
- return NULL;
+ free(res);
+
+ return NULL;
+ }
}
- }
- res->ev_cpy = ggml_metal_device_event_init(dev);
+ res->ev_cpy = ggml_metal_device_event_init(dev);
- const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
+ const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
- snprintf(res->name, sizeof(res->name), "%s", props_dev->name);
+ snprintf(res->name, sizeof(res->name), "%s", props_dev->name);
- res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
+ res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
- res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil;
- res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;
+ res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil;
+ res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;
- {
- const char * val = getenv("GGML_METAL_GRAPH_DEBUG");
- res->debug_graph = val ? atoi(val) : 0;
- }
+ {
+ const char * val = getenv("GGML_METAL_GRAPH_DEBUG");
+ res->debug_graph = val ? atoi(val) : 0;
+ }
- {
- const char * val = getenv("GGML_METAL_FUSION_DEBUG");
- res->debug_fusion = val ? atoi(val) : 0;
- }
+ {
+ const char * val = getenv("GGML_METAL_FUSION_DEBUG");
+ res->debug_fusion = val ? atoi(val) : 0;
+ }
- res->use_graph_optimize = true;
+ res->use_graph_optimize = true;
- if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {
- res->use_graph_optimize = false;
- }
+ if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {
+ res->use_graph_optimize = false;
+ }
- memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));
+ memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));
- GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false");
- GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false");
- GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");
+ GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false");
+ GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false");
+ GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");
- res->capture_compute = 0;
- res->capture_started = false;
- res->capture_scope = nil;
+ res->capture_compute = 0;
+ res->capture_started = false;
+ res->capture_scope = nil;
- {
- const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE");
- if (val) {
- res->capture_compute = atoi(val);
+ {
+ const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE");
+ if (val) {
+ res->capture_compute = atoi(val);
+ }
}
- }
- res->has_error = false;
+ res->has_error = false;
- res->gf = nil;
- res->encode_async = nil;
- for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {
- res->cmd_bufs[i].obj = nil;
- }
+ res->gf = nil;
+ res->encode_async = nil;
+ for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {
+ res->cmd_bufs[i].obj = nil;
+ }
- res->cmd_bufs_ext = [[NSMutableArray alloc] init];
+ res->cmd_bufs_ext = [[NSMutableArray alloc] init];
- res->cmd_buf_last = nil;
+ res->cmd_buf_last = nil;
- res->pipelines_ext = ggml_metal_pipelines_init();
+ res->pipelines_ext = ggml_metal_pipelines_init();
- return res;
+ return res;
+ }
}
void ggml_metal_free(ggml_metal_t ctx) {
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 19c57820e8..41ce90dc8a 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -778,7 +778,9 @@ void ggml_metal_encoder_free(ggml_metal_encoder_t encoder) {
}
void ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name) {
- [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];
+ @autoreleasepool {
+ [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];
+ }
}
void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {
@@ -1023,249 +1025,251 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
assert(dev != NULL);
- if (dev->mtl_device == nil) {
- dev->mtl_device = MTLCreateSystemDefaultDevice();
-
- if (dev->mtl_device) {
- dev->mtl_queue = [dev->mtl_device newCommandQueue];
- if (dev->mtl_queue == nil) {
- GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
- }
+ @autoreleasepool {
+ if (dev->mtl_device == nil) {
+ dev->mtl_device = MTLCreateSystemDefaultDevice();
- dev->addr_virt = 0x000000400ULL;
+ if (dev->mtl_device) {
+ dev->mtl_queue = [dev->mtl_device newCommandQueue];
+ if (dev->mtl_queue == nil) {
+ GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
+ }
- dev->props.device = device;
+ dev->addr_virt = 0x000000400ULL;
- // the Metal backend uses the system default device as the single physical device;
- // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES
- dev->props.device_phys = 0;
- dev->props.device_virt = device;
+ dev->props.device = device;
- dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
- dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
+ // the Metal backend uses the system default device as the single physical device;
+ // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES
+ dev->props.device_phys = 0;
+ dev->props.device_virt = device;
- dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
- dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;
+ dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
+ dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
- dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
- dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];
- if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {
- dev->props.has_bfloat = false;
- }
+ dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
+ dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;
- dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];
- if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {
- dev->props.has_tensor = false;
- }
+ dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
+ dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];
+ if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {
+ dev->props.has_bfloat = false;
+ }
- // note: disable the tensor API by default for old chips because with the current implementation it is not useful
- // - M2 Ultra: ~5% slower
- // - M4, M4 Max: no significant difference
- //
- // TODO: try to update the tensor API kernels to at least match the simdgroup performance
- if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&
- ![[dev->mtl_device name] containsString:@"M5"] &&
- ![[dev->mtl_device name] containsString:@"M6"] &&
- ![[dev->mtl_device name] containsString:@"A19"] &&
- ![[dev->mtl_device name] containsString:@"A20"]) {
- GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);
- dev->props.has_tensor = false;
- }
+ dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];
+ if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {
+ dev->props.has_tensor = false;
+ }
- // double-check that the tensor API compiles
- if (dev->props.has_tensor) {
- const char * src_tensor_f16 = "\n"
- "#include <metal_stdlib> \n"
- "#include <metal_tensor> \n"
- "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
- " \n"
- "using namespace metal; \n"
- "using namespace mpp::tensor_ops; \n"
- " \n"
- "kernel void dummy_kernel( \n"
- " tensor<device half, dextents<int32_t, 2>> A [[buffer(0)]], \n"
- " tensor<device half, dextents<int32_t, 2>> B [[buffer(1)]], \n"
- " device float * C [[buffer(2)]], \n"
- " uint2 tgid [[threadgroup_position_in_grid]]) \n"
- "{ \n"
- " auto tA = A.slice(0, (int)tgid.y); \n"
- " auto tB = B.slice((int)tgid.x, 0); \n"
- " \n"
- " matmul2d< \n"
- " matmul2d_descriptor(16, 16, dynamic_extent), \n"
- " execution_simdgroups<4>> mm; \n"
- " \n"
- " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
- " \n"
- " auto sA = tA.slice(0, 0); \n"
- " auto sB = tB.slice(0, 0); \n"
- " mm.run(sB, sA, cT); \n"
- " \n"
- " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
- " \n"
- " cT.store(tC); \n"
- "}";
-
- GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);
- ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);
- if (lib == NULL) {
- GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
+ // note: disable the tensor API by default for old chips because with the current implementation it is not useful
+ // - M2 Ultra: ~5% slower
+ // - M4, M4 Max: no significant difference
+ //
+ // TODO: try to update the tensor API kernels to at least match the simdgroup performance
+ if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&
+ ![[dev->mtl_device name] containsString:@"M5"] &&
+ ![[dev->mtl_device name] containsString:@"M6"] &&
+ ![[dev->mtl_device name] containsString:@"A19"] &&
+ ![[dev->mtl_device name] containsString:@"A20"]) {
+ GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);
dev->props.has_tensor = false;
- } else {
- struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
- if (!ppl.pipeline) {
+ }
+
+ // double-check that the tensor API compiles
+ if (dev->props.has_tensor) {
+ const char * src_tensor_f16 = "\n"
+ "#include <metal_stdlib> \n"
+ "#include <metal_tensor> \n"
+ "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
+ " \n"
+ "using namespace metal; \n"
+ "using namespace mpp::tensor_ops; \n"
+ " \n"
+ "kernel void dummy_kernel( \n"
+ " tensor<device half, dextents<int32_t, 2>> A [[buffer(0)]], \n"
+ " tensor<device half, dextents<int32_t, 2>> B [[buffer(1)]], \n"
+ " device float * C [[buffer(2)]], \n"
+ " uint2 tgid [[threadgroup_position_in_grid]]) \n"
+ "{ \n"
+ " auto tA = A.slice(0, (int)tgid.y); \n"
+ " auto tB = B.slice((int)tgid.x, 0); \n"
+ " \n"
+ " matmul2d< \n"
+ " matmul2d_descriptor(16, 16, dynamic_extent), \n"
+ " execution_simdgroups<4>> mm; \n"
+ " \n"
+ " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
+ " \n"
+ " auto sA = tA.slice(0, 0); \n"
+ " auto sB = tB.slice(0, 0); \n"
+ " mm.run(sB, sA, cT); \n"
+ " \n"
+ " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
+ " \n"
+ " cT.store(tC); \n"
+ "}";
+
+ GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);
+ ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);
+ if (lib == NULL) {
GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
dev->props.has_tensor = false;
- }
+ } else {
+ struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
+ if (!ppl.pipeline) {
+ GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
+ dev->props.has_tensor = false;
+ }
- ggml_metal_library_free(lib);
+ ggml_metal_library_free(lib);
+ }
}
- }
- // try to compile a dummy kernel to determine if the tensor API is supported for bfloat
- if (dev->props.has_tensor && dev->props.has_bfloat) {
- const char * src_tensor_bf16 = "\n"
- "#include <metal_stdlib> \n"
- "#include <metal_tensor> \n"
- "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
- " \n"
- "using namespace metal; \n"
- "using namespace mpp::tensor_ops; \n"
- " \n"
- "kernel void dummy_kernel( \n"
- " tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"
- " tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"
- " device float * C [[buffer(2)]], \n"
- " uint2 tgid [[threadgroup_position_in_grid]]) \n"
- "{ \n"
- " auto tA = A.slice(0, (int)tgid.y); \n"
- " auto tB = B.slice((int)tgid.x, 0); \n"
- " \n"
- " matmul2d< \n"
- " matmul2d_descriptor(16, 16, dynamic_extent), \n"
- " execution_simdgroups<4>> mm; \n"
- " \n"
- " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
- " \n"
- " auto sA = tA.slice(0, 0); \n"
- " auto sB = tB.slice(0, 0); \n"
- " mm.run(sB, sA, cT); \n"
- " \n"
- " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
- " \n"
- " cT.store(tC); \n"
- "}";
-
- GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);
- ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);
- if (lib == NULL) {
- GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
- dev->props.has_bfloat = false;
- } else {
- struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
- if (!ppl.pipeline) {
+ // try to compile a dummy kernel to determine if the tensor API is supported for bfloat
+ if (dev->props.has_tensor && dev->props.has_bfloat) {
+ const char * src_tensor_bf16 = "\n"
+ "#include <metal_stdlib> \n"
+ "#include <metal_tensor> \n"
+ "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
+ " \n"
+ "using namespace metal; \n"
+ "using namespace mpp::tensor_ops; \n"
+ " \n"
+ "kernel void dummy_kernel( \n"
+ " tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"
+ " tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"
+ " device float * C [[buffer(2)]], \n"
+ " uint2 tgid [[threadgroup_position_in_grid]]) \n"
+ "{ \n"
+ " auto tA = A.slice(0, (int)tgid.y); \n"
+ " auto tB = B.slice((int)tgid.x, 0); \n"
+ " \n"
+ " matmul2d< \n"
+ " matmul2d_descriptor(16, 16, dynamic_extent), \n"
+ " execution_simdgroups<4>> mm; \n"
+ " \n"
+ " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
+ " \n"
+ " auto sA = tA.slice(0, 0); \n"
+ " auto sB = tB.slice(0, 0); \n"
+ " mm.run(sB, sA, cT); \n"
+ " \n"
+ " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
+ " \n"
+ " cT.store(tC); \n"
+ "}";
+
+ GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);
+ ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);
+ if (lib == NULL) {
GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
dev->props.has_bfloat = false;
- }
+ } else {
+ struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
+ if (!ppl.pipeline) {
+ GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
+ dev->props.has_bfloat = false;
+ }
- ggml_metal_library_free(lib);
+ ggml_metal_library_free(lib);
+ }
}
- }
- dev->props.use_residency_sets = true;
+ dev->props.use_residency_sets = true;
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
- dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;
+ dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;
#endif
- dev->props.use_shared_buffers = dev->props.has_unified_memory;
+ dev->props.use_shared_buffers = dev->props.has_unified_memory;
#if TARGET_OS_OSX
- // In case of eGPU, shared memory may be preferable.
- dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;
+ // In case of eGPU, shared memory may be preferable.
+ dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;
#endif
- if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {
- dev->props.use_shared_buffers = false;
- }
- if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {
- dev->props.use_shared_buffers = true;
- }
+ if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {
+ dev->props.use_shared_buffers = false;
+ }
+ if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {
+ dev->props.use_shared_buffers = true;
+ }
- dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
+ dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
- dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);
+ dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);
- dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
+ dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
- dev->props.max_buffer_size = dev->mtl_device.maxBufferLength;
- dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;
- if (@available(macOS 10.12, iOS 16.0, *)) {
- dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize;
- } else {
- dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;
- }
-
- snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
- const char * gpu_name = [[dev->mtl_device name] UTF8String];
- if (n_devices > 1) {
- snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",
- gpu_name, dev->props.device_phys, dev->props.device_virt);
- } else {
- snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);
- }
+ dev->props.max_buffer_size = dev->mtl_device.maxBufferLength;
+ dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;
+ if (@available(macOS 10.12, iOS 16.0, *)) {
+ dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize;
+ } else {
+ dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;
+ }
- dev->library = ggml_metal_library_init(dev);
- if (!dev->library) {
- GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);
- }
+ snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
+ const char * gpu_name = [[dev->mtl_device name] UTF8String];
+ if (n_devices > 1) {
+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",
+ gpu_name, dev->props.device_phys, dev->props.device_virt);
+ } else {
+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);
+ }
- if (dev->props.use_residency_sets) {
- dev->rsets = ggml_metal_rsets_init(dev);
- } else {
- dev->rsets = nil;
- }
+ dev->library = ggml_metal_library_init(dev);
+ if (!dev->library) {
+ GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);
+ }
- // print MTL GPU family:
- GGML_LOG_INFO("%s: GPU name: %s (%s)\n", __func__, dev->props.name, dev->props.desc);
+ if (dev->props.use_residency_sets) {
+ dev->rsets = ggml_metal_rsets_init(dev);
+ } else {
+ dev->rsets = nil;
+ }
- // determine max supported GPU family
- // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
- // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
- {
- for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
- if ([dev->mtl_device supportsFamily:i]) {
- dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;
- GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i);
- break;
+ // print MTL GPU family:
+ GGML_LOG_INFO("%s: GPU name: %s (%s)\n", __func__, dev->props.name, dev->props.desc);
+
+ // determine max supported GPU family
+ // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
+ // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+ {
+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
+ if ([dev->mtl_device supportsFamily:i]) {
+ dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;
+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i);
+ break;
+ }
}
- }
- for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
- if ([dev->mtl_device supportsFamily:i]) {
- GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
- break;
+ for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
+ if ([dev->mtl_device supportsFamily:i]) {
+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
+ break;
+ }
}
- }
- for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {
- if ([dev->mtl_device supportsFamily:i]) {
- GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);
- break;
+ for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {
+ if ([dev->mtl_device supportsFamily:i]) {
+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);
+ break;
+ }
}
}
- }
- GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");
- GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false");
- GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false");
- GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false");
- GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false");
- GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false");
- GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false");
+ GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");
+ GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false");
+ GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false");
+ GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false");
+ GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false");
+ GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false");
+ GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false");
#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
- if (@available(macOS 10.12, iOS 16.0, *)) {
- GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);
- }
+ if (@available(macOS 10.12, iOS 16.0, *)) {
+ GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);
+ }
#endif
+ }
}
}