LLM inference in C/C++
C++ ggml
metal : fix more leaks due to missing autoreleasepools (#27883)
Niklas Wenzelfe2120bc9db2
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.mindex 1227ed39a0..e1129db302 100644--- a/ggml/src/ggml-metal/ggml-metal-context.m+++ b/ggml/src/ggml-metal/ggml-metal-context.m@@ -69,6 +69,10 @@ struct ggml_metal {// extra command buffers for things like getting, setting and copying tensorsNSMutableArray * cmd_bufs_ext;+ // buffers to release after async Metal operations complete+ // if Metal released them, it would do so on a Metal-internal thread without an autorelease pool, which could cause leaks+ NSMutableArray * buf_refs;+// the last command buffer queued into the Metal queue with operations relevant to the current Metal backendid<MTLCommandBuffer> cmd_buf_last;@@ -179,6 +183,7 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {}res->cmd_bufs_ext = [[NSMutableArray alloc] init];+ res->buf_refs = [[NSMutableArray alloc] init];res->cmd_buf_last = nil;@@ -206,6 +211,11 @@ void ggml_metal_free(ggml_metal_t ctx) {[ctx->cmd_bufs_ext removeAllObjects];[ctx->cmd_bufs_ext release];+ @autoreleasepool {+ [ctx->buf_refs removeAllObjects];+ [ctx->buf_refs release];+ }+if (ctx->pipelines_ext) {ggml_metal_pipelines_free(ctx->pipelines_ext);ctx->pipelines_ext = nil;@@ -294,6 +304,10 @@ void ggml_metal_synchronize(ggml_metal_t ctx) {[ctx->cmd_bufs_ext removeAllObjects];}++ @autoreleasepool {+ [ctx->buf_refs removeAllObjects];+ }}static struct ggml_metal_buffer_id ggml_metal_get_buffer_id(const struct ggml_tensor * t) {@@ -337,6 +351,8 @@ void ggml_metal_set_tensor_async(ggml_metal_t ctx, struct ggml_tensor * tensor,[encoder endEncoding];[cmd_buf commit];++ [ctx->buf_refs addObject:buf_src];[buf_src release];// do not wait here for completion@@ -381,6 +397,8 @@ void ggml_metal_get_tensor_async(ggml_metal_t ctx, const struct ggml_tensor * te[encoder endEncoding];[cmd_buf commit];++ [ctx->buf_refs addObject:buf_dst];[buf_dst release];// do not wait here for completiondiff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 9f2eb07313..844de31f05 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -1346,19 +1346,21 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {void ggml_metal_device_free(ggml_metal_device_t dev) {assert(dev != NULL);- ggml_metal_rsets_free(dev->rsets);+ @autoreleasepool {+ ggml_metal_rsets_free(dev->rsets);- ggml_metal_library_free(dev->library);- dev->library = NULL;+ ggml_metal_library_free(dev->library);+ dev->library = NULL;- if (dev->mtl_queue) {- [dev->mtl_queue release];- dev->mtl_queue = nil;- }+ if (dev->mtl_queue) {+ [dev->mtl_queue release];+ dev->mtl_queue = nil;+ }- if (dev->mtl_device) {- [dev->mtl_device release];- dev->mtl_device = nil;+ if (dev->mtl_device) {+ [dev->mtl_device release];+ dev->mtl_device = nil;+ }}free(dev);@@ -1446,12 +1448,14 @@ ggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) {}void ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev) {- id<MTLSharedEvent> event = ev->obj;- [event release];+ @autoreleasepool {+ id<MTLSharedEvent> event = ev->obj;+ [event release];- free(ev);+ free(ev);- GGML_UNUSED(dev);+ GGML_UNUSED(dev);+ }}void ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev) {@@ -2226,13 +2230,15 @@ ggml_metal_buffer_t ggml_metal_buffer_map(ggml_metal_device_t dev, void * ptr, s}void ggml_metal_buffer_free(ggml_metal_buffer_t buf) {- ggml_metal_device_rsets_rm(buf->dev, buf->rset);+ @autoreleasepool {+ ggml_metal_device_rsets_rm(buf->dev, buf->rset);- for (int i = 0; i < buf->n_buffers; i++) {- [buf->buffers[i].metal release];- }+ for (int i = 0; i < buf->n_buffers; i++) {+ [buf->buffers[i].metal release];+ }- ggml_metal_buffer_rset_free(buf);+ ggml_metal_buffer_rset_free(buf);+ }if (buf->is_shared && buf->owned) {#if TARGET_OS_OSX