Skip to content

LLM inference in C/C++

C++ ggml

metal: fix memory unwire if model is freed without any GPU operations (#26082)

Niklas Wenzeld0bfb1981266
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index d0956df506..91b841b67b 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -213,7 +213,7 @@ typedef void * ggml_metal_rset_t;
// a collection of residency sets (non-owning)
typedef struct ggml_metal_rsets * ggml_metal_rsets_t;
-ggml_metal_rsets_t ggml_metal_rsets_init(void);
+ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev);
void ggml_metal_rsets_free(ggml_metal_rsets_t rsets);
//
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 4edd77c6f2..7d2a686850 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -557,7 +557,32 @@ struct ggml_metal_rsets {
dispatch_group_t d_group;
};
-ggml_metal_rsets_t ggml_metal_rsets_init(void) {
+#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
+static void ggml_metal_dummy_work(ggml_metal_device_t dev) {
+ if (dev->mtl_queue == nil) {
+ return;
+ }
+
+ @autoreleasepool {
+ // perform a minimal dummy operation on the GPU
+ id<MTLBuffer> buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate];
+ id<MTLCommandBuffer> cmd_buf = [dev->mtl_queue commandBuffer];
+
+ {
+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
+
+ [encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0];
+
+ [encoder endEncoding];
+ }
+
+ [cmd_buf commit];
+ [buf release];
+ }
+}
+#endif
+
+ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) {
ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets));
res->lock = [[NSLock alloc] init];
@@ -610,6 +635,15 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) {
#endif
});
+#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
+ if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
+ // workaround for residency set memory not being released if no GPU operation occurs
+ // https://developer.apple.com/forums/thread/839089
+ // https://github.com/ggml-org/llama.cpp/issues/25937
+ ggml_metal_dummy_work(dev);
+ }
+#endif
+
return res;
}
@@ -864,7 +898,7 @@ ggml_metal_device_t ggml_metal_device_init(int device) {
}
if (dev->props.use_residency_sets) {
- dev->rsets = ggml_metal_rsets_init();
+ dev->rsets = ggml_metal_rsets_init(dev);
} else {
dev->rsets = nil;
}
@@ -1484,6 +1518,7 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {
if (buf->rset) {
[buf->rset endResidency];
[buf->rset removeAllAllocations];
+ [buf->rset commit];
[buf->rset release];
}
}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 7a93b19a07..805b744726 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -278,6 +278,9 @@ set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIRED
llama_build_and_test(test-save-load-state.cpp LABEL "model" ARGS -m "${MODEL_DEST}")
set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED test-download-model)
+if (APPLE)
+ llama_build(test-rset-release.cpp get-model.cpp)
+endif()
if (NOT GGML_BACKEND_DL)
# these tests use the backends directly and cannot be built with dynamic loading
llama_build_and_test(test-barrier.cpp)
diff --git a/tests/test-rset-release.cpp b/tests/test-rset-release.cpp
new file mode 100644
index 0000000000..bf03c5e8bd
--- /dev/null
+++ b/tests/test-rset-release.cpp
@@ -0,0 +1,53 @@
+// ref: https://github.com/ggml-org/llama.cpp/issues/25937
+// only works reliably when run with a large model that occupies 3GB+ of wired memory
+// thus, this test is not run by default
+// example model to run with: google/gemma-4-E4B-it-qat-q4_0-gguf
+
+#include <cstdint>
+#include <mach/mach.h>
+#include <mach/mach_host.h>
+#include <unistd.h>
+
+#include "llama.h"
+#include "get-model.h"
+
+static uint64_t wired_memory() {
+ vm_statistics64_data_t vmstat;
+ mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
+ if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vmstat, &count) != KERN_SUCCESS) {
+ return UINT64_MAX;
+ }
+ return static_cast<uint64_t>(vmstat.wire_count) * vm_kernel_page_size;
+}
+
+int main(int argc, char ** argv) {
+ auto * model_path = get_model_or_exit(argc, argv);
+
+ llama_backend_init();
+
+ const uint64_t wired_initial = wired_memory();
+
+ llama_model_params params = llama_model_default_params();
+ params.load_mode = LLAMA_LOAD_MODE_NONE;
+ struct llama_model* model = llama_model_load_from_file(model_path, params);
+
+ const uint64_t wired_loaded = wired_memory();
+ const uint64_t wired_delta = wired_loaded - wired_initial;
+ // system memory fluctuates, so we need to allocate enough to reliably detect the release
+ GGML_ASSERT(wired_delta > 2'000'000'000); // 2GB
+
+ llama_model_free(model);
+
+ const uint64_t t_start_ms = ggml_time_ms();
+
+ // expect most of the allocated memory to be released within 10 seconds
+ // we allow for some tolerance due to system-wide memory fluctuations
+ while (wired_memory() > wired_loaded - 0.75 * wired_delta) {
+ GGML_ASSERT(ggml_time_ms() - t_start_ms < 10'000);
+ usleep(100'000); // 100ms
+ }
+
+ llama_backend_free();
+
+ return 0;
+}