LLM inference in C/C++
C++ ggml
metal : fix memory leaks due to missing autoreleasepools (#27758)
Niklas Wenzel915dc6d38c9b
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.mindex 32d97cd5d0..1227ed39a0 100644--- a/ggml/src/ggml-metal/ggml-metal-context.m+++ b/ggml/src/ggml-metal/ggml-metal-context.m@@ -84,106 +84,108 @@ struct ggml_metal {ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {GGML_LOG_INFO("%s: allocating\n", __func__);+ @autoreleasepool {#if TARGET_OS_OSX && !GGML_METAL_NDEBUG- // Show all the Metal device instances in the system- NSArray * devices = MTLCopyAllDevices();- for (id<MTLDevice> device in devices) {- GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);- }- [devices release]; // since it was created by a *Copy* C method+ // Show all the Metal device instances in the system+ NSArray * devices = MTLCopyAllDevices();+ for (id<MTLDevice> device in devices) {+ GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);+ }+ [devices release]; // since it was created by a *Copy* C method#endif- // init context- ggml_metal_t res = calloc(1, sizeof(struct ggml_metal));-- id<MTLDevice> device = ggml_metal_device_get_obj(dev);+ // init context+ ggml_metal_t res = calloc(1, sizeof(struct ggml_metal));- GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);+ id<MTLDevice> device = ggml_metal_device_get_obj(dev);- // TODO: would it be better to have one queue for the backend and one queue for the device?- // the graph encoders and async ops would use the backend queue while the sync ops would use the device queue?- //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND]- id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev);- if (queue == nil) {- GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);- return NULL;- }+ GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);- res->dev = dev;- res->lib = ggml_metal_device_get_library(dev);- if (res->lib == NULL) {- GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__);- GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__);+ // TODO: would it be better to have one queue for the backend and one queue for the device?+ // the graph encoders and async ops would use the backend queue while the sync ops would use the device queue?+ //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND]+ id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev);+ if (queue == nil) {+ GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);+ return NULL;+ }- res->lib = ggml_metal_library_init(dev);+ res->dev = dev;+ res->lib = ggml_metal_device_get_library(dev);if (res->lib == NULL) {- GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__);+ GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__);+ GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__);- free(res);+ res->lib = ggml_metal_library_init(dev);+ if (res->lib == NULL) {+ GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__);- return NULL;+ free(res);++ return NULL;+ }}- }- res->ev_cpy = ggml_metal_device_event_init(dev);+ res->ev_cpy = ggml_metal_device_event_init(dev);- const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);+ const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);- snprintf(res->name, sizeof(res->name), "%s", props_dev->name);+ snprintf(res->name, sizeof(res->name), "%s", props_dev->name);- res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);+ res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);- res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil;- res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;+ res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil;+ res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;- {- const char * val = getenv("GGML_METAL_GRAPH_DEBUG");- res->debug_graph = val ? atoi(val) : 0;- }+ {+ const char * val = getenv("GGML_METAL_GRAPH_DEBUG");+ res->debug_graph = val ? atoi(val) : 0;+ }- {- const char * val = getenv("GGML_METAL_FUSION_DEBUG");- res->debug_fusion = val ? atoi(val) : 0;- }+ {+ const char * val = getenv("GGML_METAL_FUSION_DEBUG");+ res->debug_fusion = val ? atoi(val) : 0;+ }- res->use_graph_optimize = true;+ res->use_graph_optimize = true;- if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {- res->use_graph_optimize = false;- }+ if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {+ res->use_graph_optimize = false;+ }- memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));+ memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));- GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false");- GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false");- GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");+ GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false");+ GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false");+ GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");- res->capture_compute = 0;- res->capture_started = false;- res->capture_scope = nil;+ res->capture_compute = 0;+ res->capture_started = false;+ res->capture_scope = nil;- {- const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE");- if (val) {- res->capture_compute = atoi(val);+ {+ const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE");+ if (val) {+ res->capture_compute = atoi(val);+ }}- }- res->has_error = false;+ res->has_error = false;- res->gf = nil;- res->encode_async = nil;- for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {- res->cmd_bufs[i].obj = nil;- }+ res->gf = nil;+ res->encode_async = nil;+ for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {+ res->cmd_bufs[i].obj = nil;+ }- res->cmd_bufs_ext = [[NSMutableArray alloc] init];+ res->cmd_bufs_ext = [[NSMutableArray alloc] init];- res->cmd_buf_last = nil;+ res->cmd_buf_last = nil;- res->pipelines_ext = ggml_metal_pipelines_init();+ res->pipelines_ext = ggml_metal_pipelines_init();- return res;+ return res;+ }}void ggml_metal_free(ggml_metal_t ctx) {diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 19c57820e8..41ce90dc8a 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -778,7 +778,9 @@ void ggml_metal_encoder_free(ggml_metal_encoder_t encoder) {}void ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name) {- [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];+ @autoreleasepool {+ [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];+ }}void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {@@ -1023,249 +1025,251 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {assert(dev != NULL);- if (dev->mtl_device == nil) {- dev->mtl_device = MTLCreateSystemDefaultDevice();-- if (dev->mtl_device) {- dev->mtl_queue = [dev->mtl_device newCommandQueue];- if (dev->mtl_queue == nil) {- GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);- }+ @autoreleasepool {+ if (dev->mtl_device == nil) {+ dev->mtl_device = MTLCreateSystemDefaultDevice();- dev->addr_virt = 0x000000400ULL;+ if (dev->mtl_device) {+ dev->mtl_queue = [dev->mtl_device newCommandQueue];+ if (dev->mtl_queue == nil) {+ GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);+ }- dev->props.device = device;+ dev->addr_virt = 0x000000400ULL;- // the Metal backend uses the system default device as the single physical device;- // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES- dev->props.device_phys = 0;- dev->props.device_virt = device;+ dev->props.device = device;- dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];- dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];+ // the Metal backend uses the system default device as the single physical device;+ // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES+ dev->props.device_phys = 0;+ dev->props.device_virt = device;- dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];- dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;+ dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];+ dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];- dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];- dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];- if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {- dev->props.has_bfloat = false;- }+ dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];+ dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;- dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];- if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {- dev->props.has_tensor = false;- }+ dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];+ dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];+ if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {+ dev->props.has_bfloat = false;+ }- // note: disable the tensor API by default for old chips because with the current implementation it is not useful- // - M2 Ultra: ~5% slower- // - M4, M4 Max: no significant difference- //- // TODO: try to update the tensor API kernels to at least match the simdgroup performance- if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&- ![[dev->mtl_device name] containsString:@"M5"] &&- ![[dev->mtl_device name] containsString:@"M6"] &&- ![[dev->mtl_device name] containsString:@"A19"] &&- ![[dev->mtl_device name] containsString:@"A20"]) {- GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);- dev->props.has_tensor = false;- }+ dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];+ if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {+ dev->props.has_tensor = false;+ }- // double-check that the tensor API compiles- if (dev->props.has_tensor) {- const char * src_tensor_f16 = "\n"- "#include <metal_stdlib> \n"- "#include <metal_tensor> \n"- "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"- " \n"- "using namespace metal; \n"- "using namespace mpp::tensor_ops; \n"- " \n"- "kernel void dummy_kernel( \n"- " tensor<device half, dextents<int32_t, 2>> A [[buffer(0)]], \n"- " tensor<device half, dextents<int32_t, 2>> B [[buffer(1)]], \n"- " device float * C [[buffer(2)]], \n"- " uint2 tgid [[threadgroup_position_in_grid]]) \n"- "{ \n"- " auto tA = A.slice(0, (int)tgid.y); \n"- " auto tB = B.slice((int)tgid.x, 0); \n"- " \n"- " matmul2d< \n"- " matmul2d_descriptor(16, 16, dynamic_extent), \n"- " execution_simdgroups<4>> mm; \n"- " \n"- " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"- " \n"- " auto sA = tA.slice(0, 0); \n"- " auto sB = tB.slice(0, 0); \n"- " mm.run(sB, sA, cT); \n"- " \n"- " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"- " \n"- " cT.store(tC); \n"- "}";-- GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);- ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);- if (lib == NULL) {- GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);+ // note: disable the tensor API by default for old chips because with the current implementation it is not useful+ // - M2 Ultra: ~5% slower+ // - M4, M4 Max: no significant difference+ //+ // TODO: try to update the tensor API kernels to at least match the simdgroup performance+ if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&+ ![[dev->mtl_device name] containsString:@"M5"] &&+ ![[dev->mtl_device name] containsString:@"M6"] &&+ ![[dev->mtl_device name] containsString:@"A19"] &&+ ![[dev->mtl_device name] containsString:@"A20"]) {+ GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);dev->props.has_tensor = false;- } else {- struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);- if (!ppl.pipeline) {+ }++ // double-check that the tensor API compiles+ if (dev->props.has_tensor) {+ const char * src_tensor_f16 = "\n"+ "#include <metal_stdlib> \n"+ "#include <metal_tensor> \n"+ "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"+ " \n"+ "using namespace metal; \n"+ "using namespace mpp::tensor_ops; \n"+ " \n"+ "kernel void dummy_kernel( \n"+ " tensor<device half, dextents<int32_t, 2>> A [[buffer(0)]], \n"+ " tensor<device half, dextents<int32_t, 2>> B [[buffer(1)]], \n"+ " device float * C [[buffer(2)]], \n"+ " uint2 tgid [[threadgroup_position_in_grid]]) \n"+ "{ \n"+ " auto tA = A.slice(0, (int)tgid.y); \n"+ " auto tB = B.slice((int)tgid.x, 0); \n"+ " \n"+ " matmul2d< \n"+ " matmul2d_descriptor(16, 16, dynamic_extent), \n"+ " execution_simdgroups<4>> mm; \n"+ " \n"+ " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"+ " \n"+ " auto sA = tA.slice(0, 0); \n"+ " auto sB = tB.slice(0, 0); \n"+ " mm.run(sB, sA, cT); \n"+ " \n"+ " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"+ " \n"+ " cT.store(tC); \n"+ "}";++ GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);+ ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);+ if (lib == NULL) {GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);dev->props.has_tensor = false;- }+ } else {+ struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);+ if (!ppl.pipeline) {+ GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);+ dev->props.has_tensor = false;+ }- ggml_metal_library_free(lib);+ ggml_metal_library_free(lib);+ }}- }- // try to compile a dummy kernel to determine if the tensor API is supported for bfloat- if (dev->props.has_tensor && dev->props.has_bfloat) {- const char * src_tensor_bf16 = "\n"- "#include <metal_stdlib> \n"- "#include <metal_tensor> \n"- "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"- " \n"- "using namespace metal; \n"- "using namespace mpp::tensor_ops; \n"- " \n"- "kernel void dummy_kernel( \n"- " tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"- " tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"- " device float * C [[buffer(2)]], \n"- " uint2 tgid [[threadgroup_position_in_grid]]) \n"- "{ \n"- " auto tA = A.slice(0, (int)tgid.y); \n"- " auto tB = B.slice((int)tgid.x, 0); \n"- " \n"- " matmul2d< \n"- " matmul2d_descriptor(16, 16, dynamic_extent), \n"- " execution_simdgroups<4>> mm; \n"- " \n"- " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"- " \n"- " auto sA = tA.slice(0, 0); \n"- " auto sB = tB.slice(0, 0); \n"- " mm.run(sB, sA, cT); \n"- " \n"- " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"- " \n"- " cT.store(tC); \n"- "}";-- GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);- ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);- if (lib == NULL) {- GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);- dev->props.has_bfloat = false;- } else {- struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);- if (!ppl.pipeline) {+ // try to compile a dummy kernel to determine if the tensor API is supported for bfloat+ if (dev->props.has_tensor && dev->props.has_bfloat) {+ const char * src_tensor_bf16 = "\n"+ "#include <metal_stdlib> \n"+ "#include <metal_tensor> \n"+ "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"+ " \n"+ "using namespace metal; \n"+ "using namespace mpp::tensor_ops; \n"+ " \n"+ "kernel void dummy_kernel( \n"+ " tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"+ " tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"+ " device float * C [[buffer(2)]], \n"+ " uint2 tgid [[threadgroup_position_in_grid]]) \n"+ "{ \n"+ " auto tA = A.slice(0, (int)tgid.y); \n"+ " auto tB = B.slice((int)tgid.x, 0); \n"+ " \n"+ " matmul2d< \n"+ " matmul2d_descriptor(16, 16, dynamic_extent), \n"+ " execution_simdgroups<4>> mm; \n"+ " \n"+ " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"+ " \n"+ " auto sA = tA.slice(0, 0); \n"+ " auto sB = tB.slice(0, 0); \n"+ " mm.run(sB, sA, cT); \n"+ " \n"+ " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"+ " \n"+ " cT.store(tC); \n"+ "}";++ GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);+ ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);+ if (lib == NULL) {GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);dev->props.has_bfloat = false;- }+ } else {+ struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);+ if (!ppl.pipeline) {+ GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);+ dev->props.has_bfloat = false;+ }- ggml_metal_library_free(lib);+ ggml_metal_library_free(lib);+ }}- }- dev->props.use_residency_sets = true;+ dev->props.use_residency_sets = true;#if defined(GGML_METAL_HAS_RESIDENCY_SETS)- dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;+ dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;#endif- dev->props.use_shared_buffers = dev->props.has_unified_memory;+ dev->props.use_shared_buffers = dev->props.has_unified_memory;#if TARGET_OS_OSX- // In case of eGPU, shared memory may be preferable.- dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;+ // In case of eGPU, shared memory may be preferable.+ dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;#endif- if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {- dev->props.use_shared_buffers = false;- }- if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {- dev->props.use_shared_buffers = true;- }+ if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {+ dev->props.use_shared_buffers = false;+ }+ if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {+ dev->props.use_shared_buffers = true;+ }- dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];+ dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];- dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);+ dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);- dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;+ dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;- dev->props.max_buffer_size = dev->mtl_device.maxBufferLength;- dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;- if (@available(macOS 10.12, iOS 16.0, *)) {- dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize;- } else {- dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;- }-- snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);- const char * gpu_name = [[dev->mtl_device name] UTF8String];- if (n_devices > 1) {- snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",- gpu_name, dev->props.device_phys, dev->props.device_virt);- } else {- snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);- }+ dev->props.max_buffer_size = dev->mtl_device.maxBufferLength;+ dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;+ if (@available(macOS 10.12, iOS 16.0, *)) {+ dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize;+ } else {+ dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;+ }- dev->library = ggml_metal_library_init(dev);- if (!dev->library) {- GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);- }+ snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);+ const char * gpu_name = [[dev->mtl_device name] UTF8String];+ if (n_devices > 1) {+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",+ gpu_name, dev->props.device_phys, dev->props.device_virt);+ } else {+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);+ }- if (dev->props.use_residency_sets) {- dev->rsets = ggml_metal_rsets_init(dev);- } else {- dev->rsets = nil;- }+ dev->library = ggml_metal_library_init(dev);+ if (!dev->library) {+ GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);+ }- // print MTL GPU family:- GGML_LOG_INFO("%s: GPU name: %s (%s)\n", __func__, dev->props.name, dev->props.desc);+ if (dev->props.use_residency_sets) {+ dev->rsets = ggml_metal_rsets_init(dev);+ } else {+ dev->rsets = nil;+ }- // determine max supported GPU family- // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf- // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf- {- for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {- if ([dev->mtl_device supportsFamily:i]) {- dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;- GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i);- break;+ // print MTL GPU family:+ GGML_LOG_INFO("%s: GPU name: %s (%s)\n", __func__, dev->props.name, dev->props.desc);++ // determine max supported GPU family+ // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf+ // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf+ {+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {+ if ([dev->mtl_device supportsFamily:i]) {+ dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i);+ break;+ }}- }- for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {- if ([dev->mtl_device supportsFamily:i]) {- GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);- break;+ for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {+ if ([dev->mtl_device supportsFamily:i]) {+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);+ break;+ }}- }- for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {- if ([dev->mtl_device supportsFamily:i]) {- GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);- break;+ for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {+ if ([dev->mtl_device supportsFamily:i]) {+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);+ break;+ }}}- }- GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");- GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false");- GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false");- GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false");- GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false");- GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false");- GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false");+ GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");+ GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false");+ GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false");+ GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false");+ GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false");+ GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false");+ GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false");#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)- if (@available(macOS 10.12, iOS 16.0, *)) {- GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);- }+ if (@available(macOS 10.12, iOS 16.0, *)) {+ GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);+ }#endif+ }}}