Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Avoid crashing with BF16 on Apple Metal
Justine Tunney82f845c4fa77
diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex c519a797..87f21714 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -757,6 +757,12 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_tensor * t, size_t * offs}static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) {+ for (size_t i = 0, n = 3; i < n; ++i) {+ if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {+ return false;+ }+ }+switch (op->op) {case GGML_OP_UNARY:switch (ggml_get_unary_op(op)) {