LLM inference in C/C++
C++ ggml
metal: enable Metal 4.0 tensor API on M5+/A19+ (#27461)
James Francisd5d993a0938d
diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txtindex 140c5d809e..2094a409f9 100644--- a/ggml/src/ggml-metal/CMakeLists.txt+++ b/ggml/src/ggml-metal/CMakeLists.txt@@ -163,19 +163,37 @@ else())endforeach()+ # the tensor API kernels go in a separate metallib, loaded only where supported+ set(AIR_MM_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mul_mm_tensor.air")+ add_custom_command(+ OUTPUT ${AIR_MM_TENSOR}+ COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/mul_mm.metal -o ${AIR_MM_TENSOR}+ DEPENDS kernels/mul_mm.metal kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h+ COMMENT "Compiling kernels/mul_mm.metal (tensor API)"+ VERBATIM+ )++ add_custom_command(+ OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib+ COMMAND xcrun -sdk macosx metallib ${AIR_MM_TENSOR} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib+ DEPENDS ${AIR_MM_TENSOR}+ COMMENT "Linking tensor API Metal kernels into ggml-tensor.metallib"+ )+add_custom_command(OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallibCOMMAND xcrun -sdk macosx metallib ${AIR_FILES} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallibCOMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.hCOMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.hCOMMAND rm -rf ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels- DEPENDS ${AIR_FILES}+ DEPENDS ${AIR_FILES} ${AIR_MM_TENSOR}COMMENT "Linking Metal kernels into default.metallib")add_custom_target(ggml-metal-lib ALLDEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib+ ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib)endif() # GGML_METAL_EMBED_LIBRARY@@ -188,6 +206,7 @@ if (NOT GGML_METAL_EMBED_LIBRARY)install(FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib+ ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallibDESTINATION ${CMAKE_INSTALL_BINDIR})endif()diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 0d8484d002..9f2eb07313 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -27,6 +27,9 @@static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;static const NSInteger MTLGPUFamilyMetal4_GGML = 5002;+// MTLLanguageVersion4_0 is not present in older SDKs+static const NSUInteger MTLLanguageVersion4_0_GGML = 4 << 16;+#if !GGML_METAL_EMBED_LIBRARY// Here to assist with NSBundle Path Hack@interface GGMLMetalClass : NSObject@@ -154,6 +157,9 @@ struct ggml_metal_library {// nil in single_library mode (everything resolves to objs[0]).NSMutableDictionary<NSString *, NSNumber *> * fn_to_lib;+ // kernels from a second metallib, resolved ahead of the combined library+ NSSet<NSString *> * override_fns;+ggml_metal_device_t dev;ggml_metal_pipelines_t pipelines; // cache of compiled pipelines@@ -174,6 +180,18 @@ static void ggml_metal_library_build_index(ggml_metal_library_t lib) {}}+// note: defined below, after struct ggml_metal_device+static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev);++// the tensor API headers are exposed to the shader compiler only at Metal language version 4.0+static void ggml_metal_compile_options_set_lang(MTLCompileOptions * options, bool has_tensor) {+ if (!has_tensor) {+ return;+ }++ options.languageVersion = (MTLLanguageVersion) MTLLanguageVersion4_0_GGML;+}+// Parse a `#include "name"` line. Returns the quoted name in *include_name on// success. Whitespace-tolerant; ignores `#include <...>` (system headers).static bool ggml_metal_library_parse_quoted_include(NSString * line, NSString ** include_name) {@@ -313,6 +331,7 @@ static bool ggml_metal_library_compile_all(@autoreleasepool {MTLCompileOptions * options = [MTLCompileOptions new];options.preprocessorMacros = prep;+ ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(res->dev)->has_tensor);lib = [device newLibraryWithSource:src options:options error:&error];@@ -369,6 +388,46 @@ static bool ggml_metal_library_compile_all(return ok;}+// look for <name>.metallib as a bundle resource, then next to the running binary+static NSString * ggml_metal_find_metallib(NSBundle * bundle, NSString * name) {+ NSError * error = nil;++ NSString * path_lib = [bundle pathForResource:name ofType:@"metallib"];+ if (path_lib == nil) {+ // Try to find the resource in the directory where the current binary located.+ NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];+ NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];++ NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, [name stringByAppendingPathExtension:@"metallib"]]];+ if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {+ GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);++ NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];+ if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {+ // Optionally, if this is a symlink, try to resolve it.+ path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];+ if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {+ // It is a relative path, adding the binary directory as directory prefix.+ path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];+ }+ if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {+ // Link to the resource could not be resolved.+ path_lib_default = nil;+ } else {+ GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);+ }+ }+ } else {+ // The resource couldn't be found in the binary's directory.+ path_lib_default = nil;+ }++ path_lib = path_lib_default;+ }++ return path_lib;+}+ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {id<MTLDevice> device = ggml_metal_device_get_obj(dev);@@ -432,38 +491,7 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {const int64_t t_start = ggml_time_us();NSError * error = nil;- NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];- if (path_lib == nil) {- // Try to find the resource in the directory where the current binary located.- NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];- NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];-- NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, @"default.metallib"]];- if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {- GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);-- NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];- if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {- // Optionally, if this is a symlink, try to resolve it.- path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];- if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {- // It is a relative path, adding the binary directory as directory prefix.- path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];- }- if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {- // Link to the resource could not be resolved.- path_lib_default = nil;- } else {- GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);- }- }- } else {- // The resource couldn't be found in the binary's directory.- path_lib_default = nil;- }-- path_lib = path_lib_default;- }+ NSString * path_lib = ggml_metal_find_metallib(bundle, @"default");if (path_lib != nil) {// pre-compiled library found: a single combined default.metallib@@ -478,6 +506,30 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {return NULL;}+ // the tensor API kernels are built into a separate metallib+ if (ggml_metal_device_get_props(dev)->has_tensor) {+ NSString * path_mm = ggml_metal_find_metallib(bundle, @"ggml-tensor");++ id<MTLLibrary> lib_mm = nil;+ if (path_mm != nil) {+ lib_mm = [device newLibraryWithURL:[NSURL fileURLWithPath:path_mm] error:&error];+ if (!lib_mm && error) {+ GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);+ }+ }++ if (lib_mm) {+ GGML_LOG_INFO("%s: loaded '%s'\n", __func__, [path_mm UTF8String]);++ res->objs[GGML_METAL_LIB_MUL_MM] = [lib_mm retain];+ res->override_fns = [[NSSet setWithArray:[lib_mm functionNames]] retain];+ } else {+ GGML_LOG_INFO("%s: ggml-tensor.metallib not found - disabling the tensor API\n", __func__);++ ggml_metal_device_disable_tensor(dev);+ }+ }+GGML_LOG_INFO("%s: loaded in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);return res;}@@ -557,6 +609,7 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t devMTLCompileOptions * options = [MTLCompileOptions new];options.preprocessorMacros = prep;+ ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(dev)->has_tensor);library = [device newLibraryWithSource:src options:options error:&error];if (error) {@@ -615,6 +668,10 @@ void ggml_metal_library_free(ggml_metal_library_t lib) {[lib->fn_to_lib release];}+ if (lib->override_fns) {+ [lib->override_fns release];+ }+ggml_metal_pipelines_free(lib->pipelines);[lib->lock release];@@ -676,7 +733,9 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_// route to the library that actually defines this kernel; fn_to_lib is// built from -[MTLLibrary functionNames] so it's always in syncint lib_idx = 0;- if (!lib->single_library) {+ if (lib->override_fns && [lib->override_fns containsObject:base_func]) {+ lib_idx = GGML_METAL_LIB_MUL_MM;+ } else if (!lib->single_library) {NSNumber * idx = lib->fn_to_lib[base_func];if (!idx) {[lib->lock unlock];@@ -1862,6 +1921,10 @@ const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_dereturn &dev->props;}+static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {+ dev->props.has_tensor = false;+}+//// device buffers//diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txtindex fe3d14ffc5..c46377c762 100644--- a/tests/CMakeLists.txt+++ b/tests/CMakeLists.txt@@ -299,6 +299,14 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)endif()llama_build_and_test(test-backend-ops.cpp)+# the tensor API kernels come from a separate metallib - check they produce correct results+# ref: https://github.com/ggml-org/llama.cpp/issues/27473+if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY)+ llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor+ ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K)+ set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1)+endif()+llama_build_and_test(test-model-load-cancel.cpp LABEL "model")llama_build_and_test(test-autorelease.cpp LABEL "model")llama_build_and_test(test-backend-sampler.cpp LABEL "model")