Skip to content

LLM inference in C/C++

C++ ggml

metal : fix more leaks due to missing autoreleasepools (#27883)

Niklas Wenzelfe2120bc9db2
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m
index 1227ed39a0..e1129db302 100644
--- a/ggml/src/ggml-metal/ggml-metal-context.m
+++ b/ggml/src/ggml-metal/ggml-metal-context.m
@@ -69,6 +69,10 @@ struct ggml_metal {
// extra command buffers for things like getting, setting and copying tensors
NSMutableArray * cmd_bufs_ext;
+ // buffers to release after async Metal operations complete
+ // if Metal released them, it would do so on a Metal-internal thread without an autorelease pool, which could cause leaks
+ NSMutableArray * buf_refs;
+
// the last command buffer queued into the Metal queue with operations relevant to the current Metal backend
id<MTLCommandBuffer> cmd_buf_last;
@@ -179,6 +183,7 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
}
res->cmd_bufs_ext = [[NSMutableArray alloc] init];
+ res->buf_refs = [[NSMutableArray alloc] init];
res->cmd_buf_last = nil;
@@ -206,6 +211,11 @@ void ggml_metal_free(ggml_metal_t ctx) {
[ctx->cmd_bufs_ext removeAllObjects];
[ctx->cmd_bufs_ext release];
+ @autoreleasepool {
+ [ctx->buf_refs removeAllObjects];
+ [ctx->buf_refs release];
+ }
+
if (ctx->pipelines_ext) {
ggml_metal_pipelines_free(ctx->pipelines_ext);
ctx->pipelines_ext = nil;
@@ -294,6 +304,10 @@ void ggml_metal_synchronize(ggml_metal_t ctx) {
[ctx->cmd_bufs_ext removeAllObjects];
}
+
+ @autoreleasepool {
+ [ctx->buf_refs removeAllObjects];
+ }
}
static struct ggml_metal_buffer_id ggml_metal_get_buffer_id(const struct ggml_tensor * t) {
@@ -337,6 +351,8 @@ void ggml_metal_set_tensor_async(ggml_metal_t ctx, struct ggml_tensor * tensor,
[encoder endEncoding];
[cmd_buf commit];
+
+ [ctx->buf_refs addObject:buf_src];
[buf_src release];
// do not wait here for completion
@@ -381,6 +397,8 @@ void ggml_metal_get_tensor_async(ggml_metal_t ctx, const struct ggml_tensor * te
[encoder endEncoding];
[cmd_buf commit];
+
+ [ctx->buf_refs addObject:buf_dst];
[buf_dst release];
// do not wait here for completion
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 9f2eb07313..844de31f05 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -1346,19 +1346,21 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
void ggml_metal_device_free(ggml_metal_device_t dev) {
assert(dev != NULL);
- ggml_metal_rsets_free(dev->rsets);
+ @autoreleasepool {
+ ggml_metal_rsets_free(dev->rsets);
- ggml_metal_library_free(dev->library);
- dev->library = NULL;
+ ggml_metal_library_free(dev->library);
+ dev->library = NULL;
- if (dev->mtl_queue) {
- [dev->mtl_queue release];
- dev->mtl_queue = nil;
- }
+ if (dev->mtl_queue) {
+ [dev->mtl_queue release];
+ dev->mtl_queue = nil;
+ }
- if (dev->mtl_device) {
- [dev->mtl_device release];
- dev->mtl_device = nil;
+ if (dev->mtl_device) {
+ [dev->mtl_device release];
+ dev->mtl_device = nil;
+ }
}
free(dev);
@@ -1446,12 +1448,14 @@ ggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) {
}
void ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev) {
- id<MTLSharedEvent> event = ev->obj;
- [event release];
+ @autoreleasepool {
+ id<MTLSharedEvent> event = ev->obj;
+ [event release];
- free(ev);
+ free(ev);
- GGML_UNUSED(dev);
+ GGML_UNUSED(dev);
+ }
}
void ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev) {
@@ -2226,13 +2230,15 @@ ggml_metal_buffer_t ggml_metal_buffer_map(ggml_metal_device_t dev, void * ptr, s
}
void ggml_metal_buffer_free(ggml_metal_buffer_t buf) {
- ggml_metal_device_rsets_rm(buf->dev, buf->rset);
+ @autoreleasepool {
+ ggml_metal_device_rsets_rm(buf->dev, buf->rset);
- for (int i = 0; i < buf->n_buffers; i++) {
- [buf->buffers[i].metal release];
- }
+ for (int i = 0; i < buf->n_buffers; i++) {
+ [buf->buffers[i].metal release];
+ }
- ggml_metal_buffer_rset_free(buf);
+ ggml_metal_buffer_rset_free(buf);
+ }
if (buf->is_shared && buf->owned) {
#if TARGET_OS_OSX