LLM inference in C/C++
C++ ggml
metal: fix memory unwire if model is freed without any GPU operations (#26082)
Niklas Wenzeld0bfb1981266
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex d0956df506..91b841b67b 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -213,7 +213,7 @@ typedef void * ggml_metal_rset_t;// a collection of residency sets (non-owning)typedef struct ggml_metal_rsets * ggml_metal_rsets_t;-ggml_metal_rsets_t ggml_metal_rsets_init(void);+ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev);void ggml_metal_rsets_free(ggml_metal_rsets_t rsets);//diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 4edd77c6f2..7d2a686850 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -557,7 +557,32 @@ struct ggml_metal_rsets {dispatch_group_t d_group;};-ggml_metal_rsets_t ggml_metal_rsets_init(void) {+#if defined(GGML_METAL_HAS_RESIDENCY_SETS)+static void ggml_metal_dummy_work(ggml_metal_device_t dev) {+ if (dev->mtl_queue == nil) {+ return;+ }++ @autoreleasepool {+ // perform a minimal dummy operation on the GPU+ id<MTLBuffer> buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate];+ id<MTLCommandBuffer> cmd_buf = [dev->mtl_queue commandBuffer];++ {+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];++ [encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0];++ [encoder endEncoding];+ }++ [cmd_buf commit];+ [buf release];+ }+}+#endif++ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) {ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets));res->lock = [[NSLock alloc] init];@@ -610,6 +635,15 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) {#endif});+#if defined(GGML_METAL_HAS_RESIDENCY_SETS)+ if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {+ // workaround for residency set memory not being released if no GPU operation occurs+ // https://developer.apple.com/forums/thread/839089+ // https://github.com/ggml-org/llama.cpp/issues/25937+ ggml_metal_dummy_work(dev);+ }+#endif+return res;}@@ -864,7 +898,7 @@ ggml_metal_device_t ggml_metal_device_init(int device) {}if (dev->props.use_residency_sets) {- dev->rsets = ggml_metal_rsets_init();+ dev->rsets = ggml_metal_rsets_init(dev);} else {dev->rsets = nil;}@@ -1484,6 +1518,7 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {if (buf->rset) {[buf->rset endResidency];[buf->rset removeAllAllocations];+ [buf->rset commit];[buf->rset release];}}diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txtindex 7a93b19a07..805b744726 100644--- a/tests/CMakeLists.txt+++ b/tests/CMakeLists.txt@@ -278,6 +278,9 @@ set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIREDllama_build_and_test(test-save-load-state.cpp LABEL "model" ARGS -m "${MODEL_DEST}")set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED test-download-model)+if (APPLE)+ llama_build(test-rset-release.cpp get-model.cpp)+endif()if (NOT GGML_BACKEND_DL)# these tests use the backends directly and cannot be built with dynamic loadingllama_build_and_test(test-barrier.cpp)diff --git a/tests/test-rset-release.cpp b/tests/test-rset-release.cppnew file mode 100644index 0000000000..bf03c5e8bd--- /dev/null+++ b/tests/test-rset-release.cpp@@ -0,0 +1,53 @@+// ref: https://github.com/ggml-org/llama.cpp/issues/25937+// only works reliably when run with a large model that occupies 3GB+ of wired memory+// thus, this test is not run by default+// example model to run with: google/gemma-4-E4B-it-qat-q4_0-gguf++#include <cstdint>+#include <mach/mach.h>+#include <mach/mach_host.h>+#include <unistd.h>++#include "llama.h"+#include "get-model.h"++static uint64_t wired_memory() {+ vm_statistics64_data_t vmstat;+ mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;+ if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vmstat, &count) != KERN_SUCCESS) {+ return UINT64_MAX;+ }+ return static_cast<uint64_t>(vmstat.wire_count) * vm_kernel_page_size;+}++int main(int argc, char ** argv) {+ auto * model_path = get_model_or_exit(argc, argv);++ llama_backend_init();++ const uint64_t wired_initial = wired_memory();++ llama_model_params params = llama_model_default_params();+ params.load_mode = LLAMA_LOAD_MODE_NONE;+ struct llama_model* model = llama_model_load_from_file(model_path, params);++ const uint64_t wired_loaded = wired_memory();+ const uint64_t wired_delta = wired_loaded - wired_initial;+ // system memory fluctuates, so we need to allocate enough to reliably detect the release+ GGML_ASSERT(wired_delta > 2'000'000'000); // 2GB++ llama_model_free(model);++ const uint64_t t_start_ms = ggml_time_ms();++ // expect most of the allocated memory to be released within 10 seconds+ // we allow for some tolerance due to system-wide memory fluctuations+ while (wired_memory() > wired_loaded - 0.75 * wired_delta) {+ GGML_ASSERT(ggml_time_ms() - t_start_ms < 10'000);+ usleep(100'000); // 100ms+ }++ llama_backend_free();++ return 0;+}