Skip to content

LLM inference in C/C++

C++ ggml

metal: enable Metal 4.0 tensor API on M5+/A19+ (#27461)

James Francisd5d993a0938d
diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt
index 140c5d809e..2094a409f9 100644
--- a/ggml/src/ggml-metal/CMakeLists.txt
+++ b/ggml/src/ggml-metal/CMakeLists.txt
@@ -163,19 +163,37 @@ else()
)
endforeach()
+ # the tensor API kernels go in a separate metallib, loaded only where supported
+ set(AIR_MM_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mul_mm_tensor.air")
+ add_custom_command(
+ OUTPUT ${AIR_MM_TENSOR}
+ COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/mul_mm.metal -o ${AIR_MM_TENSOR}
+ DEPENDS kernels/mul_mm.metal kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h
+ COMMENT "Compiling kernels/mul_mm.metal (tensor API)"
+ VERBATIM
+ )
+
+ add_custom_command(
+ OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
+ COMMAND xcrun -sdk macosx metallib ${AIR_MM_TENSOR} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
+ DEPENDS ${AIR_MM_TENSOR}
+ COMMENT "Linking tensor API Metal kernels into ggml-tensor.metallib"
+ )
+
add_custom_command(
OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
COMMAND xcrun -sdk macosx metallib ${AIR_FILES} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h
COMMAND rm -rf ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels
- DEPENDS ${AIR_FILES}
+ DEPENDS ${AIR_FILES} ${AIR_MM_TENSOR}
COMMENT "Linking Metal kernels into default.metallib"
)
add_custom_target(
ggml-metal-lib ALL
DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
+ ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
)
endif() # GGML_METAL_EMBED_LIBRARY
@@ -188,6 +206,7 @@ if (NOT GGML_METAL_EMBED_LIBRARY)
install(
FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
+ ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
DESTINATION ${CMAKE_INSTALL_BINDIR}
)
endif()
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 0d8484d002..9f2eb07313 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -27,6 +27,9 @@
static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;
static const NSInteger MTLGPUFamilyMetal4_GGML = 5002;
+// MTLLanguageVersion4_0 is not present in older SDKs
+static const NSUInteger MTLLanguageVersion4_0_GGML = 4 << 16;
+
#if !GGML_METAL_EMBED_LIBRARY
// Here to assist with NSBundle Path Hack
@interface GGMLMetalClass : NSObject
@@ -154,6 +157,9 @@ struct ggml_metal_library {
// nil in single_library mode (everything resolves to objs[0]).
NSMutableDictionary<NSString *, NSNumber *> * fn_to_lib;
+ // kernels from a second metallib, resolved ahead of the combined library
+ NSSet<NSString *> * override_fns;
+
ggml_metal_device_t dev;
ggml_metal_pipelines_t pipelines; // cache of compiled pipelines
@@ -174,6 +180,18 @@ static void ggml_metal_library_build_index(ggml_metal_library_t lib) {
}
}
+// note: defined below, after struct ggml_metal_device
+static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev);
+
+// the tensor API headers are exposed to the shader compiler only at Metal language version 4.0
+static void ggml_metal_compile_options_set_lang(MTLCompileOptions * options, bool has_tensor) {
+ if (!has_tensor) {
+ return;
+ }
+
+ options.languageVersion = (MTLLanguageVersion) MTLLanguageVersion4_0_GGML;
+}
+
// Parse a `#include "name"` line. Returns the quoted name in *include_name on
// success. Whitespace-tolerant; ignores `#include <...>` (system headers).
static bool ggml_metal_library_parse_quoted_include(NSString * line, NSString ** include_name) {
@@ -313,6 +331,7 @@ static bool ggml_metal_library_compile_all(
@autoreleasepool {
MTLCompileOptions * options = [MTLCompileOptions new];
options.preprocessorMacros = prep;
+ ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(res->dev)->has_tensor);
lib = [device newLibraryWithSource:src options:options error:&error];
@@ -369,6 +388,46 @@ static bool ggml_metal_library_compile_all(
return ok;
}
+// look for <name>.metallib as a bundle resource, then next to the running binary
+static NSString * ggml_metal_find_metallib(NSBundle * bundle, NSString * name) {
+ NSError * error = nil;
+
+ NSString * path_lib = [bundle pathForResource:name ofType:@"metallib"];
+ if (path_lib == nil) {
+ // Try to find the resource in the directory where the current binary located.
+ NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
+ NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
+
+ NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, [name stringByAppendingPathExtension:@"metallib"]]];
+ if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
+ GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
+
+ NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
+ if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
+ // Optionally, if this is a symlink, try to resolve it.
+ path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
+ if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
+ // It is a relative path, adding the binary directory as directory prefix.
+ path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
+ }
+ if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
+ // Link to the resource could not be resolved.
+ path_lib_default = nil;
+ } else {
+ GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
+ }
+ }
+ } else {
+ // The resource couldn't be found in the binary's directory.
+ path_lib_default = nil;
+ }
+
+ path_lib = path_lib_default;
+ }
+
+ return path_lib;
+}
+
ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
id<MTLDevice> device = ggml_metal_device_get_obj(dev);
@@ -432,38 +491,7 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
const int64_t t_start = ggml_time_us();
NSError * error = nil;
- NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];
- if (path_lib == nil) {
- // Try to find the resource in the directory where the current binary located.
- NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
- NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
-
- NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, @"default.metallib"]];
- if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
- GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
-
- NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
- if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
- // Optionally, if this is a symlink, try to resolve it.
- path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
- if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
- // It is a relative path, adding the binary directory as directory prefix.
- path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
- }
- if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
- // Link to the resource could not be resolved.
- path_lib_default = nil;
- } else {
- GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
- }
- }
- } else {
- // The resource couldn't be found in the binary's directory.
- path_lib_default = nil;
- }
-
- path_lib = path_lib_default;
- }
+ NSString * path_lib = ggml_metal_find_metallib(bundle, @"default");
if (path_lib != nil) {
// pre-compiled library found: a single combined default.metallib
@@ -478,6 +506,30 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
return NULL;
}
+ // the tensor API kernels are built into a separate metallib
+ if (ggml_metal_device_get_props(dev)->has_tensor) {
+ NSString * path_mm = ggml_metal_find_metallib(bundle, @"ggml-tensor");
+
+ id<MTLLibrary> lib_mm = nil;
+ if (path_mm != nil) {
+ lib_mm = [device newLibraryWithURL:[NSURL fileURLWithPath:path_mm] error:&error];
+ if (!lib_mm && error) {
+ GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
+ }
+ }
+
+ if (lib_mm) {
+ GGML_LOG_INFO("%s: loaded '%s'\n", __func__, [path_mm UTF8String]);
+
+ res->objs[GGML_METAL_LIB_MUL_MM] = [lib_mm retain];
+ res->override_fns = [[NSSet setWithArray:[lib_mm functionNames]] retain];
+ } else {
+ GGML_LOG_INFO("%s: ggml-tensor.metallib not found - disabling the tensor API\n", __func__);
+
+ ggml_metal_device_disable_tensor(dev);
+ }
+ }
+
GGML_LOG_INFO("%s: loaded in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);
return res;
}
@@ -557,6 +609,7 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev
MTLCompileOptions * options = [MTLCompileOptions new];
options.preprocessorMacros = prep;
+ ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(dev)->has_tensor);
library = [device newLibraryWithSource:src options:options error:&error];
if (error) {
@@ -615,6 +668,10 @@ void ggml_metal_library_free(ggml_metal_library_t lib) {
[lib->fn_to_lib release];
}
+ if (lib->override_fns) {
+ [lib->override_fns release];
+ }
+
ggml_metal_pipelines_free(lib->pipelines);
[lib->lock release];
@@ -676,7 +733,9 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_
// route to the library that actually defines this kernel; fn_to_lib is
// built from -[MTLLibrary functionNames] so it's always in sync
int lib_idx = 0;
- if (!lib->single_library) {
+ if (lib->override_fns && [lib->override_fns containsObject:base_func]) {
+ lib_idx = GGML_METAL_LIB_MUL_MM;
+ } else if (!lib->single_library) {
NSNumber * idx = lib->fn_to_lib[base_func];
if (!idx) {
[lib->lock unlock];
@@ -1862,6 +1921,10 @@ const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_de
return &dev->props;
}
+static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {
+ dev->props.has_tensor = false;
+}
+
//
// device buffers
//
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index fe3d14ffc5..c46377c762 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -299,6 +299,14 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
endif()
llama_build_and_test(test-backend-ops.cpp)
+# the tensor API kernels come from a separate metallib - check they produce correct results
+# ref: https://github.com/ggml-org/llama.cpp/issues/27473
+if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY)
+ llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor
+ ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K)
+ set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1)
+endif()
+
llama_build_and_test(test-model-load-cancel.cpp LABEL "model")
llama_build_and_test(test-autorelease.cpp LABEL "model")
llama_build_and_test(test-backend-sampler.cpp LABEL "model")