Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Separate Mozilla code from llama.cpp folder
Justine Tunneyb9fcc21ef16e
diff --git a/LICENSE b/LICENSEindex 73e7a3b0..7364c1eb 100644--- a/LICENSE+++ b/LICENSE@@ -11,32 +11,3 @@ distributed under the License is distributed on an "AS IS" BASIS,WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.See the License for the specific language governing permissions andlimitations under the License.------------------------------------------------------------------------------------MIT License--Copyright (c) 2023 Georgi Gerganov-Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.-Copyright (c) 2023 Yuji Hirose-Copyright (c) 2022 Niels Lohmann <https://nlohmann.me>-Copyright (c) 2022 Bjoern Hoehrmann <[email protected]>-Copyright (c) 2017 Sean Barrett--Permission is hereby granted, free of charge, to any person obtaining a copy-of this software and associated documentation files (the "Software"), to deal-in the Software without restriction, including without limitation the rights-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-copies of the Software, and to permit persons to whom the Software is-furnished to do so, subject to the following conditions:--The above copyright notice and this permission notice shall be included in all-copies or substantial portions of the Software.--THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE-SOFTWARE.diff --git a/Makefile b/Makefileindex 8cf1b9ff..342dc5fc 100644--- a/Makefile+++ b/Makefile@@ -11,12 +11,13 @@ MAKEFLAGS += --no-builtin-rulesinclude build/config.mkinclude build/rules.mk+include llamafile/BUILD.mkinclude llama.cpp/BUILD.mk# the root package is `o//` by default# building a package also builds its sub-packages.PHONY: o/$(MODE)/-o/$(MODE)/: o/$(MODE)/llama.cpp+o/$(MODE)/: o/$(MODE)/llama.cpp o/$(MODE)/llamafileinclude build/deps.mkinclude build/tags.mkdiff --git a/build/config.mk b/build/config.mkindex c8cca517..c68bb3ad 100644--- a/build/config.mk+++ b/build/config.mk@@ -10,7 +10,7 @@ CXX = $(PREFIX)c++ARFLAGS = rcsDCCFLAGS = -g -O3TARGET_ARCH = -Xx86_64,-msse3,-mavx,-mavx2,-mf16c,-mfma-CPPFLAGS += -iquote.+CPPFLAGS += -iquote. -mcosmoTMPDIR = o//tmpIGNORE := $(shell mkdir -p $(TMPDIR))diff --git a/build/rules.mk b/build/rules.mkindex b4330141..10fa7299 100644--- a/build/rules.mk+++ b/build/rules.mk@@ -16,6 +16,10 @@ o/$(MODE)/%.o: %.cc@mkdir -p $(@D)$(COMPILE.cc) -o $@ $<+o/$(MODE)/%.o: %.cpp+ @mkdir -p $(@D)+ $(COMPILE.cc) -o $@ $<+o/$(MODE)/%: o/$(MODE)/%.o$(LINK.o) $^ $(LOADLIBES) $(LDLIBS) -o $@diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mkindex 7db74d34..7c0c3c99 100644--- a/llama.cpp/BUILD.mk+++ b/llama.cpp/BUILD.mk@@ -7,12 +7,13 @@ LLAMA_CPP_FILES := $(wildcard llama.cpp/*.*)LLAMA_CPP_HDRS = $(filter %.h,$(LLAMA_CPP_FILES))LLAMA_CPP_INCS = $(filter %.inc,$(LLAMA_CPP_FILES))LLAMA_CPP_SRCS_C = $(filter %.c,$(LLAMA_CPP_FILES))-LLAMA_CPP_SRCS_CC = $(filter %.cc,$(LLAMA_CPP_FILES))-LLAMA_CPP_SRCS = $(LLAMA_CPP_SRCS_C) $(LLAMA_CPP_SRCS_CC)+LLAMA_CPP_SRCS_CPP = $(filter %.cpp,$(LLAMA_CPP_FILES))+LLAMA_CPP_SRCS = $(LLAMA_CPP_SRCS_C) $(LLAMA_CPP_SRCS_CPP)LLAMA_CPP_OBJS = \+ $(LLAMAFILE_OBJS) \$(LLAMA_CPP_SRCS_C:%.c=o/$(MODE)/%.o) \- $(LLAMA_CPP_SRCS_CC:%.cc=o/$(MODE)/%.o) \+ $(LLAMA_CPP_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \$(LLAMA_CPP_FILES:%=o/$(MODE)/%.zip.o)o/$(MODE)/llama.cpp/llama.cpp.a: $(LLAMA_CPP_OBJS)diff --git a/llama.cpp/LICENSE b/llama.cpp/LICENSEnew file mode 100644index 00000000..83b046d8--- /dev/null+++ b/llama.cpp/LICENSE@@ -0,0 +1,26 @@+MIT License++Copyright (c) 2023 Georgi Gerganov+Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.+Copyright (c) 2023 Yuji Hirose+Copyright (c) 2022 Niels Lohmann <https://nlohmann.me>+Copyright (c) 2022 Bjoern Hoehrmann <[email protected]>+Copyright (c) 2017 Sean Barrett++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex a7ee16be..47a499d0 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -4,18 +4,10 @@ DESCRIPTIONLICENSE- llama.cpp is licensed MIT and in our repository it includes files- released under the Apache 2.0 license. Files that are licensed as- such are clearly documented at the top of each file.+ MITORIGINhttps://github.com/ggerganov/llama.cppe9c1cecb9d7d743d30b4a29ecd56a411437def0aTue Nov 7 09:04:51 2023 +0100--LOCAL CHANGES-- - Added #ifdef guards around C++ exception usage- - Unified weights i/o by introducing ggml_file interface- - Added ability to mmap() weights directly from ZIP filediff --git a/llama.cpp/build-info.cc b/llama.cpp/build-info.cppsimilarity index 100%rename from llama.cpp/build-info.ccrename to llama.cpp/build-info.cppdiff --git a/llama.cpp/common.cc b/llama.cpp/common.cppsimilarity index 100%rename from llama.cpp/common.ccrename to llama.cpp/common.cppdiff --git a/llama.cpp/console.cc b/llama.cpp/console.cppsimilarity index 100%rename from llama.cpp/console.ccrename to llama.cpp/console.cppdiff --git a/llama.cpp/cuda.c b/llama.cpp/cuda.cindex 89682f90..e69de29b 100644--- a/llama.cpp/cuda.c+++ b/llama.cpp/cuda.c@@ -1,433 +0,0 @@-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2023 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#define _COSMO_SOURCE-#include <cosmo.h>-#include <time.h>-#include <dlfcn.h>-#include <errno.h>-#include <spawn.h>-#include <assert.h>-#include <unistd.h>-#include <stdlib.h>-#include <limits.h>-#include <signal.h>-#include <pthread.h>-#include <sys/wait.h>-#include <sys/stat.h>-#include <stdatomic.h>-#include "ggml-cuda.h"--__static_yoink("llama.cpp/ggml.h");-__static_yoink("llama.cpp/ggml-cuda.h");-__static_yoink("llama.cpp/ggml-cuda.cu");--#define NVCC_LIBS "-lcublas"--#define NVCC_FLAGS "--shared", \- "--forward-unknown-to-host-compiler", \- "-use_fast_math", \- "-arch=compute_87", \- "--compiler-options", "-fPIC -O3 -march=native -mtune=native", \- "-DNDEBUG", \- "-DGGML_BUILD=1", \- "-DGGML_SHARED=1", \- "-DGGML_CUDA_DMMV_X=32", \- "-DGGML_CUDA_MMV_Y=1", \- "-DK_QUANTS_PER_ITERATION=2", \- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \- "-DGGML_USE_CUBLAS"--static const struct Source {- const char *zip;- const char *name;-} srcs[] = {- {"/zip/llama.cpp/ggml.h", "ggml.h"},- {"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},- {"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last-};--static struct Cuda {- bool disabled;- bool supported;- atomic_uint once;- typeof(ggml_init_cublas) *init;- typeof(ggml_cublas_loaded) *loaded;- typeof(ggml_cuda_host_free) *host_free;- typeof(ggml_cuda_host_malloc) *host_malloc;- typeof(ggml_cuda_can_mul_mat) *can_mul_mat;- typeof(ggml_cuda_set_tensor_split) *set_tensor_split;- typeof(ggml_cuda_transform_tensor) *transform_tensor;- typeof(ggml_cuda_free_data) *free_data;- typeof(ggml_cuda_assign_buffers) *assign_buffers;- typeof(ggml_cuda_assign_buffers_no_scratch) *assign_buffers_no_scratch;- typeof(ggml_cuda_assign_buffers_force_inplace) *assign_buffers_force_inplace;- typeof(ggml_cuda_assign_buffers_no_alloc) *assign_buffers_no_alloc;- typeof(ggml_cuda_assign_scratch_offset) *assign_scratch_offset;- typeof(ggml_cuda_copy_to_device) *copy_to_device;- typeof(ggml_cuda_set_main_device) *set_main_device;- typeof(ggml_cuda_set_scratch_size) *set_scratch_size;- typeof(ggml_cuda_free_scratch) *free_scratch;- typeof(ggml_cuda_compute_forward) *compute_forward;- typeof(ggml_cuda_get_device_count) *get_device_count;- typeof(ggml_cuda_get_device_description) *get_device_description;-} ggml_cuda;--static const char *Dlerror(void) {- const char *msg;- msg = cosmo_dlerror();- if (!msg) msg = "null dlopen error";- return msg;-}--static const char *GetDsoExtension(void) {- if (IsWindows()) {- return "dll";- } else if (IsXnu()) {- return "dylib";- } else {- return "so";- }-}--static bool FileExists(const char *path) {- struct stat st;- return !stat(path, &st);-}--static bool IsExecutable(const char *path) {- struct stat st;- return !stat(path, &st) &&- (st.st_mode & 0111) &&- !S_ISDIR(st.st_mode);-}--static bool CreateTempPath(const char *path, char tmp[static PATH_MAX]) {- int fd;- strlcpy(tmp, path, PATH_MAX);- strlcat(tmp, ".XXXXXX", PATH_MAX);- if ((fd = mkstemp(tmp)) != -1) {- close(fd);- return true;- } else {- perror(tmp);- return false;- }-}--static bool Compile(const char *src,- const char *tmp,- const char *out,- char *args[]) {- int pid, ws;- errno_t err = posix_spawnp(&pid, args[0], NULL, NULL, args, environ);- if (err) {- perror(args[0]);- unlink(tmp);- return false;- }- while (waitpid(pid, &ws, 0) == -1) {- if (errno != EINTR) {- perror(args[0]);- unlink(tmp);- return false;- }- }- if (ws) {- tinyprint(2, args[0], ": returned nonzero exit status\n", NULL);- unlink(tmp);- return false;- }- if (rename(tmp, out)) {- perror(out);- unlink(tmp);- return false;- }- return true;-}--// finds nvidia compiler-//-// 1. $CUDA_PATH/bin/nvcc-// 2. nvcc on $PATH environ-// 3. /opt/cuda/bin/nvcc-// 4. /usr/local/cuda/bin/nvcc-//-// set $CUDA_PATH to empty string to disable cuda-static bool GetNvccPath(char path[static PATH_MAX]) {- const char *cuda_path;- if ((cuda_path = getenv("CUDA_PATH"))) {- if (!*cuda_path) return false;- strlcpy(path, cuda_path, PATH_MAX);- strlcat(path, "/bin/", PATH_MAX);- } else if (commandv(IsWindows() ? "nvcc.exe" : "nvcc", path, PATH_MAX)) {- return true;- } else if (FileExists("/opt/cuda")) {- strlcpy(path, "/opt/cuda/bin/", PATH_MAX);- } else {- strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);- }- strlcat(path, "nvcc", PATH_MAX);- if (IsWindows()) {- strlcat(path, ".exe", PATH_MAX);- }- return IsExecutable(path);-}--static bool CompileNativeCuda(char dso[static PATH_MAX]) {-- // find full path of nvidia compiler- char nvcc[PATH_MAX];- if (!GetNvccPath(nvcc)) {- tinyprint(2, "warning: couldn't find nvcc (nvidia c compiler) "- "try setting $CUDA_PATH if it's installed\n", NULL);- return false;- }-- // extract source code- char src[PATH_MAX];- bool needs_rebuild = false;- for (int i = 0; i < sizeof(srcs) / sizeof(*srcs); ++i) {- ggml_get_app_dir(src, sizeof(src));- if (!i && mkdir(src, 0755) && errno != EEXIST) {- perror(src);- return false;- }- strlcat(src, srcs[i].name, sizeof(src));- switch (ggml_is_file_newer_than(srcs[i].zip, src)) {- case -1:- return false;- case false:- break;- case true:- needs_rebuild = true;- if (!ggml_extract(srcs[i].zip, src)) {- return false;- }- break;- default:- __builtin_unreachable();- }- }-- // check if dso is already compiled- ggml_get_app_dir(dso, PATH_MAX);- strlcat(dso, "ggml-cuda.", PATH_MAX);- strlcat(dso, GetDsoExtension(), PATH_MAX);- if (!needs_rebuild) {- switch (ggml_is_file_newer_than(src, dso)) {- case -1:- return false;- case false:- return true;- case true:- break;- default:- __builtin_unreachable();- }- }-- // create temporary output path for atomicity- char tmpdso[PATH_MAX];- if (!CreateTempPath(dso, tmpdso)) {- return false;- }-- // try building dso with host nvidia microarchitecture- tinyprint(2, "building ggml-cuda with nvcc...\n", NULL);- if (!Compile(src, tmpdso, dso, (char *[]){- nvcc, NVCC_FLAGS, "-o", tmpdso, src, NVCC_LIBS, NULL})) {- return false;- }-- // we're good!- return true;-}--static bool ImportCudaImpl(void) {-- // get native cuda dll if possible- char dso[PATH_MAX];- if (!CompileNativeCuda(dso)) {- return false;- }-- // runtime link dynamic shared object- void *lib;- lib = cosmo_dlopen(dso, RTLD_LAZY);- if (!lib) {- tinyprint(2, Dlerror(), ": failed to load library\n", NULL);- return false;- }-- // import functions- bool ok = true;- ok &= !!(ggml_cuda.init = cosmo_dlsym(lib, "ggml_init_cublas"));- ok &= !!(ggml_cuda.loaded = cosmo_dlsym(lib, "ggml_cublas_loaded"));- ok &= !!(ggml_cuda.host_free = cosmo_dlsym(lib, "ggml_cuda_host_free"));- ok &= !!(ggml_cuda.host_malloc = cosmo_dlsym(lib, "ggml_cuda_host_malloc"));- ok &= !!(ggml_cuda.can_mul_mat = cosmo_dlsym(lib, "ggml_cuda_can_mul_mat"));- ok &= !!(ggml_cuda.set_tensor_split = cosmo_dlsym(lib, "ggml_cuda_set_tensor_split"));- ok &= !!(ggml_cuda.transform_tensor = cosmo_dlsym(lib, "ggml_cuda_transform_tensor"));- ok &= !!(ggml_cuda.free_data = cosmo_dlsym(lib, "ggml_cuda_free_data"));- ok &= !!(ggml_cuda.assign_buffers = cosmo_dlsym(lib, "ggml_cuda_assign_buffers"));- ok &= !!(ggml_cuda.assign_buffers_no_scratch = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_no_scratch"));- ok &= !!(ggml_cuda.assign_buffers_force_inplace = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_force_inplace"));- ok &= !!(ggml_cuda.assign_buffers_no_alloc = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_no_alloc"));- ok &= !!(ggml_cuda.assign_scratch_offset = cosmo_dlsym(lib, "ggml_cuda_assign_scratch_offset"));- ok &= !!(ggml_cuda.copy_to_device = cosmo_dlsym(lib, "ggml_cuda_copy_to_device"));- ok &= !!(ggml_cuda.set_main_device = cosmo_dlsym(lib, "ggml_cuda_set_main_device"));- ok &= !!(ggml_cuda.set_scratch_size = cosmo_dlsym(lib, "ggml_cuda_set_scratch_size"));- ok &= !!(ggml_cuda.free_scratch = cosmo_dlsym(lib, "ggml_cuda_free_scratch"));- ok &= !!(ggml_cuda.compute_forward = cosmo_dlsym(lib, "ggml_cuda_compute_forward"));- ok &= !!(ggml_cuda.get_device_count = cosmo_dlsym(lib, "ggml_cuda_get_device_count"));- ok &= !!(ggml_cuda.get_device_description = cosmo_dlsym(lib, "ggml_cuda_get_device_description"));- if (!ok) {- tinyprint(2, Dlerror(), ": not all symbols could be imported\n", NULL);- return false;- }-- // we're good- return true;-}--static void ImportCuda(void) {- if (ggml_cuda.disabled) {- return;- }- if (ImportCudaImpl()) {- ggml_cuda.supported = true;- tinyprint(2, "NVIDIA cuBLAS GPU supported locked and loaded\n", NULL);- } else {- tinyprint(2, "warning: couldn't load Nvidia CUDA GPU support\n", NULL);- }-}--void ggml_cuda_disable(void) {- ggml_cuda.disabled = true;-}--bool ggml_cuda_supported(void) {- cosmo_once(&ggml_cuda.once, ImportCuda);- return ggml_cuda.supported;-}--void ggml_init_cublas(void) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.init();-}--bool ggml_cublas_loaded(void) {- if (!ggml_cuda_supported()) return false;- return ggml_cuda.loaded();-}--void *ggml_cuda_host_malloc(size_t n) {- if (!ggml_cuda_supported()) return NULL;- return ggml_cuda.host_malloc(n);-}--void ggml_cuda_host_free(void *data) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.host_free(data);-}--bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0,- const struct ggml_tensor *src1,- struct ggml_tensor *dst) {- if (!ggml_cuda_supported()) return false;- return ggml_cuda.can_mul_mat(src0, src1, dst);-}--void ggml_cuda_set_tensor_split(const float *tensor_split) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.set_tensor_split(tensor_split);-}--void ggml_cuda_transform_tensor(void *data, struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.transform_tensor(data, tensor);-}--void ggml_cuda_free_data(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.free_data(tensor);-}--void ggml_cuda_assign_buffers(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers(tensor);-}--void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers_no_scratch(tensor);-}--void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers_force_inplace(tensor);-}--void ggml_cuda_assign_buffers_no_alloc(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers_no_alloc(tensor);-}--void ggml_cuda_assign_scratch_offset(struct ggml_tensor *tensor, size_t offset) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_scratch_offset(tensor, offset);-}--void ggml_cuda_copy_to_device(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.copy_to_device(tensor);-}--void ggml_cuda_set_main_device(int main_device) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.set_main_device(main_device);-}--void ggml_cuda_set_scratch_size(size_t scratch_size) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.set_scratch_size(scratch_size);-}--void ggml_cuda_free_scratch(void) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.free_scratch();-}--bool ggml_cuda_compute_forward(struct ggml_compute_params *params,- struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return false;- return ggml_cuda.compute_forward(params, tensor);-}--int ggml_cuda_get_device_count(void) {- if (!ggml_cuda_supported()) return 0;- return ggml_cuda.get_device_count();-}--void ggml_cuda_get_device_description(int device,- char *description,- size_t description_size) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.get_device_description(device, description,- description_size);-}diff --git a/llama.cpp/ggml-opencl.cpp b/llama.cpp/ggml-opencl.cppdeleted file mode 100644index 202bcb48..00000000--- a/llama.cpp/ggml-opencl.cpp+++ /dev/null@@ -1,1899 +0,0 @@-#include "ggml-opencl.h"--#include <array>-#include <atomic>-#include <sstream>-#include <vector>-#include <limits>--#define CL_TARGET_OPENCL_VERSION 110-#include <clblast.h>--#include <stdlib.h>-#include <stdio.h>-#include <string.h>--#include "ggml.h"--#if defined(_MSC_VER)-#pragma warning(disable: 4244 4267) // possible loss of data-#endif--#define CL_DMMV_LOCAL_SIZE 32--#ifndef K_QUANTS_PER_ITERATION-#define K_QUANTS_PER_ITERATION 1-#else-static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUANTS_PER_ITERATION must be 1 or 2");-#endif--#define MULTILINE_QUOTE(...) #__VA_ARGS__-static std::string program_source = MULTILINE_QUOTE(--typedef char int8_t;-typedef uchar uint8_t;-typedef short int16_t;-typedef ushort uint16_t;-typedef int int32_t;-typedef uint uint32_t;--struct __attribute__ ((packed)) block_q4_0-{- half d;- uint8_t qs[QK4_0 / 2];-};--struct __attribute__ ((packed)) block_q4_1-{- half d;- half m;- uint8_t qs[QK4_1 / 2];-};--struct __attribute__ ((packed)) block_q5_0-{- half d;- uint32_t qh;- uint8_t qs[QK5_0 / 2];-};--struct __attribute__ ((packed)) block_q5_1-{- half d;- half m;- uint32_t qh;- uint8_t qs[QK5_1 / 2];-};--struct __attribute__ ((packed)) block_q8_0-{- half d;- int8_t qs[QK8_0];-};--struct __attribute__((packed)) block_q2_K-{- uint8_t scales[16];- uint8_t qs[64];- half d;- half dmin;-};--struct __attribute__((packed)) block_q3_K-{- uint8_t hmask[32];- uint8_t qs[64];- uint8_t scales[12];- half d;-};--struct __attribute__((packed)) block_q4_K-{- half d;- half dmin;- uint8_t scales[12];- uint8_t qs[128];-};--struct __attribute__((packed)) block_q5_K-{- half d;- half dmin;- uint8_t scales[12];- uint8_t qh[32];- uint8_t qs[128];-};--struct __attribute__((packed)) block_q6_K-{- uint8_t ql[128];- uint8_t qh[64];- int8_t scales[16];- half d;-};--__kernel void convert_fp16_to_fp32(__global half* x, __global float* y) {- const uint i = get_global_id(0);-- y[i] = vload_half(0, &x[i]);-}--void dequantize_q4_0(__global const struct block_q4_0* x, const int ib, const int iqs, float* v0, float* v1) {- const float d = vload_half(0, &x[ib].d);-- const uint8_t vui = x[ib].qs[iqs];-- const int8_t vi0 = vui & 0xF;- const int8_t vi1 = vui >> 4;-- *v0 = (vi0 - 8)*d;- *v1 = (vi1 - 8)*d;-}-void dequantize_q4_1(__global const struct block_q4_1* x, const int ib, const int iqs, float* v0, float* v1) {- const float d = vload_half(0, &x[ib].d);- const float m = vload_half(0, &x[ib].m);-- const uint8_t vui = x[ib].qs[iqs];-- const int8_t vi0 = vui & 0xF;- const int8_t vi1 = vui >> 4;-- *v0 = vi0*d + m;- *v1 = vi1*d + m;-}-void dequantize_q5_0(__global const struct block_q5_0* x, const int ib, const int iqs, float* v0, float* v1) {- const float d = vload_half(0, &x[ib].d);-- uint32_t qh = x[ib].qh;-- const uint8_t xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const uint8_t xh_1 = ((qh >> (iqs + 12)) ) & 0x10;-- const int32_t x0 = ((x[ib].qs[iqs] & 0xf) | xh_0) - 16;- const int32_t x1 = ((x[ib].qs[iqs] >> 4) | xh_1) - 16;-- *v0 = x0*d;- *v1 = x1*d;-}-void dequantize_q5_1(__global const struct block_q5_1* x, const int ib, const int iqs, float* v0, float* v1) {- const float d = vload_half(0, &x[ib].d);- const float m = vload_half(0, &x[ib].m);-- uint32_t qh = x[ib].qh;-- const uint8_t xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const uint8_t xh_1 = ((qh >> (iqs + 12)) ) & 0x10;-- const int32_t x0 = ((x[ib].qs[iqs] & 0xf) | xh_0);- const int32_t x1 = ((x[ib].qs[iqs] >> 4) | xh_1);-- *v0 = x0*d + m;- *v1 = x1*d + m;-}-void dequantize_q8_0(__global const struct block_q8_0* x, const int ib, const int iqs, float* v0, float* v1) {- const float d = vload_half(0, &x[ib].d);-- const int8_t vi0 = x[ib].qs[iqs + 0];- const int8_t vi1 = x[ib].qs[iqs + 1];-- *v0 = vi0*d;- *v1 = vi1*d;-}-void convert_f16(__global half* x, const int ib, const int iqs, float* v0, float* v1){- *v0 = vload_half(0, &x[ib + 0]);- *v1 = vload_half(0, &x[ib + 1]);-}-);--static std::string k_quants_source = MULTILINE_QUOTE(-inline void get_scale_min_k4(int j, const __global uint8_t *q, uint8_t *d, uint8_t *m)-{- if (j < 4)- {- *d = q[j] & 63;- *m = q[j + 4] & 63;- }- else- {- *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);- *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);- }-}--__kernel void dequantize_block_q2_K(__global const struct block_q2_K *x, __global float *yy)-{- const int i = get_group_id(0) + get_global_offset(0);- const int tid = get_local_id(0);- const int n = tid / 32;- const int l = tid - 32 * n;- const int is = 8 * n + l / 16;-- const uint8_t q = x[i].qs[32 * n + l];- __global float *y = yy + get_group_id(0) * QK_K + 128 * n;-- const float dall = vload_half(0, &x[i].d);- const float dmin = vload_half(0, &x[i].dmin);-- y[l + 0] = dall * (x[i].scales[is + 0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is + 0] >> 4);- y[l + 32] = dall * (x[i].scales[is + 2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is + 2] >> 4);- y[l + 64] = dall * (x[i].scales[is + 4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is + 4] >> 4);- y[l + 96] = dall * (x[i].scales[is + 6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is + 6] >> 4);-}--__kernel void dequantize_block_q3_K(__global const struct block_q3_K *x, __global float *yy)-{- int r = get_local_id(0) / 4;- int i = get_group_id(0) + get_global_offset(0);- int tid = r / 2;- int is0 = r % 2;- int l0 = 16 * is0 + 4 * (get_local_id(0) % 4);- int n = tid / 4;- int j = tid - 4 * n;-- uint8_t m = 1 << (4 * n + j);- int is = 8 * n + 2 * j + is0;- int shift = 2 * j;-- int8_t us = is < 4 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 8] >> 0) & 3) << 4)- : is < 8 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 4] >> 2) & 3) << 4)- : is < 12 ? (x[i].scales[is - 8] >> 4) | (((x[i].scales[is + 0] >> 4) & 3) << 4)- : (x[i].scales[is - 8] >> 4) | (((x[i].scales[is - 4] >> 6) & 3) << 4);- float d_all = vload_half(0, &x[i].d);- float dl = d_all * (us - 32);-- __global float *y = yy + get_group_id(0) * QK_K + 128 * n + 32 * j;- const __global uint8_t *q = x[i].qs + 32 * n;- const __global uint8_t *hm = x[i].hmask;-- for (int l = l0; l < l0 + 4; ++l)- y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));-}--__kernel void dequantize_block_q4_K(__global const struct block_q4_K *x, __global float *yy)-{- const int i = get_group_id(0) + get_global_offset(0);- const int tid = get_local_id(0);- const int il = tid / 8;- const int ir = tid % 8;- const int is = 2 * il;- const int n = 4;-- __global float *y = yy + get_group_id(0) * QK_K + 64 * il + n * ir;-- const float dall = vload_half(0, &x[i].d);- const float dmin = vload_half(0, &x[i].dmin);-- __global const uint8_t *q = x[i].qs + 32 * il + n * ir;-- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, &sc, &m);- float d1 = dall * sc;- float m1 = dmin * m;- get_scale_min_k4(is + 1, x[i].scales, &sc, &m);- float d2 = dall * sc;- float m2 = dmin * m;- for (int l = 0; l < n; ++l)- {- y[l + 0] = d1 * (q[l] & 0xF) - m1;- y[l + 32] = d2 * (q[l] >> 4) - m2;- }-}--__kernel void dequantize_block_q5_K(__global const struct block_q5_K *x, __global float *yy)-{- const int i = get_group_id(0) + get_global_offset(0);- const int tid = get_local_id(0);- const int il = tid / 16;- const int ir = tid % 16;- const int is = 2 * il;-- __global float *y = yy + get_group_id(0) * QK_K + 64 * il + 2 * ir;-- const float dall = vload_half(0, &x[i].d);- const float dmin = vload_half(0, &x[i].dmin);-- __global const uint8_t *ql = x[i].qs + 32 * il + 2 * ir;- __global const uint8_t *qh = x[i].qh + 2 * ir;-- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, &sc, &m);- const float d1 = dall * sc;- const float m1 = dmin * m;- get_scale_min_k4(is + 1, x[i].scales, &sc, &m);- const float d2 = dall * sc;- const float m2 = dmin * m;-- uint8_t hm = 1 << (2 * il);- y[0] = d1 * ((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0)) - m1;- y[1] = d1 * ((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0)) - m1;- hm <<= 1;- y[32] = d2 * ((ql[0] >> 4) + (qh[0] & hm ? 16 : 0)) - m2;- y[33] = d2 * ((ql[1] >> 4) + (qh[1] & hm ? 16 : 0)) - m2;-}--__kernel void dequantize_block_q6_K(__global const struct block_q6_K *x, __global float *yy)-{- const int i = get_group_id(0) + get_global_offset(0);- const int tid = get_local_id(0);- const int ip = tid / 32;- const int il = tid - 32 * ip;- const int is = 8 * ip + il / 16;-- __global float *y = yy + get_group_id(0) * QK_K + 128 * ip + il;-- const float d = vload_half(0, &x[i].d);-- __global const uint8_t *ql = x[i].ql + 64 * ip + il;- const uint8_t qh = x[i].qh[32 * ip + il];- __global const int8_t *sc = x[i].scales + is;-- y[0] = d * sc[0] * ((int8_t)((ql[0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);- y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);- y[64] = d * sc[4] * ((int8_t)((ql[0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);- y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);-}--__kernel void dequantize_mul_mat_vec_q2_K(__global const struct block_q2_K * xx, __local float* tmp, __global float* yy, __global float* dst, const int ncols) {-- const int row = get_group_id(0);-- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row + get_global_offset(0);-- __global const struct block_q2_K * x = xx + ib0;-- const int tid = get_local_id(0)/K_QUANTS_PER_ITERATION; // 0...31 or 0...15- const int ix = get_local_id(0)%K_QUANTS_PER_ITERATION; // 0 or 0,1-- const int step = 16/K_QUANTS_PER_ITERATION;-- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0...15 or 0...7-- const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2- const int q_offset = 32*im + l0;- const int s_offset = 8*im;- const int y_offset = 128*im + l0;-- tmp[16 * ix + tid] = 0;-- uint32_t aux[4];- const uint8_t * d = (const uint8_t *)aux;- const uint8_t * m = (const uint8_t *)(aux + 2);-- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {-- __global const float * y = yy + i * QK_K + y_offset;- __global const uint8_t * q = x[i].qs + q_offset;-- const float dall = vload_half(0, &x[i].d);- const float dmin = vload_half(0, &x[i].dmin);-- __global const uint32_t * a = (__global const uint32_t *)(x[i].scales + s_offset);- aux[0] = a[0] & 0x0f0f0f0f;- aux[1] = a[1] & 0x0f0f0f0f;- aux[2] = (a[0] >> 4) & 0x0f0f0f0f;- aux[3] = (a[1] >> 4) & 0x0f0f0f0f;-- float sum1 = 0, sum2 = 0;- for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {- sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)- + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)- + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)- + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)- + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)- + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)- + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)- +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);- sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]- + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];-- }- tmp[16 * ix + tid] += dall * sum1 - dmin * sum2;-- }-- // sum up partial sums and write back result- barrier(CLK_LOCAL_MEM_FENCE);- for (int s=16; s>0; s>>=1) {- if (tid < s) {- tmp[tid] += tmp[tid + s];- }- barrier(CLK_LOCAL_MEM_FENCE);- }- if (tid == 0) {- dst[row] = tmp[0];- }-}--__kernel void dequantize_mul_mat_vec_q3_K(__global const struct block_q3_K * xx, __local float* tmp, __global float* yy, __global float* dst, const int ncols) {- const uint16_t kmask1 = 0x0303;- const uint16_t kmask2 = 0x0f0f;-- const int row = get_group_id(0);-- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row + get_global_offset(0);-- __global const struct block_q3_K * x = xx + ib0;-- const int tid = get_local_id(0)/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = get_local_id(0)%K_QUANTS_PER_ITERATION; // 0 or 0,1-- const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop- const int step = 16/K_QUANTS_PER_ITERATION;- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0....15 or 0...7-- const uint8_t m = 1 << (4*im);-- const int l0 = n*in; // 0...15 or 0...14 in steps of 2- const int q_offset = 32*im + l0;- const int y_offset = 128*im + l0;-- uint16_t utmp[4];- const int8_t * s = (const int8_t *)utmp;-- const uint16_t s_shift = 4*im;-- tmp[16 * ix + tid] = 0;-- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {-- __global const float * y = yy + i * QK_K + y_offset;- __global const uint8_t * q = x[i].qs + q_offset;- __global const uint8_t * h = x[i].hmask + l0;-- __global const uint16_t * a = (__global const uint16_t *)x[i].scales;- utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);- utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);- utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);- utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);-- const float d = vload_half(0, &x[i].d);-- float sum = 0;- for (int l = 0; l < n; ++l) {- sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))- + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))- + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))- + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));- sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))- + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))- + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))- + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));- }- tmp[16 * ix + tid] += d * sum;-- }-- // sum up partial sums and write back result- barrier(CLK_LOCAL_MEM_FENCE);- for (int s=16; s>0; s>>=1) {- if (tid < s) {- tmp[tid] += tmp[tid + s];- }- barrier(CLK_LOCAL_MEM_FENCE);- }- if (tid == 0) {- dst[row] = tmp[0];- }-}--__kernel void dequantize_mul_mat_vec_q4_K(__global const struct block_q4_K * xx, __local float* tmp, __global float* yy, __global float* dst, const int ncols) {-- //to rename it later, just to test now- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;-- const int row = get_group_id(0);- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row + get_global_offset(0);-- const int tid = get_local_id(0)/K_QUANTS_PER_ITERATION; // 0...15- const int ix = get_local_id(0)%K_QUANTS_PER_ITERATION;-- const int step = 8/K_QUANTS_PER_ITERATION;-- const int il = tid/step; // 0...3- const int ir = tid - step*il;// 0...3- const int n = 2*K_QUANTS_PER_ITERATION;-- const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224- const int in = il%2;-- const int l0 = n*(2*ir + in);- const int q_offset = 32*im + l0;- const int y_offset = 64*im + l0;-- uint16_t aux[4];- const uint8_t * sc = (const uint8_t *)aux;-- __global const struct block_q4_K * x = xx + ib0;-- tmp[16 * ix + tid] = 0;-- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {-- __global const uint8_t * q1 = x[i].qs + q_offset;- __global const uint8_t * q2 = q1 + 64;- __global const float * y1 = yy + i*QK_K + y_offset;- __global const float * y2 = y1 + 128;-- const float dall = vload_half(0, &x[i].d);- const float dmin = vload_half(0, &x[i].dmin);-- __global const uint16_t * a = (__global const uint16_t *)x[i].scales;- aux[0] = a[im+0] & kmask1;- aux[1] = a[im+2] & kmask1;- aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);- aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);-- float4 s = (float4)(0.f);- float smin = 0;- for (int l = 0; l < n; ++l) {- s.x += y1[l] * (q1[l] & 0xF); s.y += y1[l+32] * (q1[l] >> 4);- s.z += y2[l] * (q2[l] & 0xF); s.w += y2[l+32] * (q2[l] >> 4);- smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];- }- tmp[16 * ix + tid] += dall * (s.x * sc[0] + s.y * sc[1] + s.z * sc[4] + s.w * sc[5]) - dmin * smin;-- }-- // sum up partial sums and write back result- barrier(CLK_LOCAL_MEM_FENCE);- for (int s=16; s>0; s>>=1) {- if (tid < s) {- tmp[tid] += tmp[tid + s];- }- barrier(CLK_LOCAL_MEM_FENCE);- }- if (tid == 0) {- dst[row] = tmp[0];- }-}--__kernel void dequantize_mul_mat_vec_q5_K(__global const struct block_q5_K * xx, __local float* tmp, __global float* yy, __global float* dst, const int ncols) {-- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;-- const int row = get_group_id(0);- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row + get_global_offset(0);-- const int tid = get_local_id(0)/2; // 0...15- const int ix = get_local_id(0)%2;-- const int il = tid/4; // 0...3- const int ir = tid - 4*il;// 0...3- const int n = 2;-- const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224- const int in = il%2;-- const int l0 = n*(2*ir + in);- const int q_offset = 32*im + l0;- const int y_offset = 64*im + l0;-- const uint8_t hm1 = 1 << (2*im);- const uint8_t hm2 = hm1 << 4;-- uint16_t aux[4];- const uint8_t * sc = (const uint8_t *)aux;-- __global const struct block_q5_K * x = xx + ib0;-- tmp[16 * ix + tid] = 0;-- for (int i = ix; i < num_blocks_per_row; i += 2) {-- __global const uint8_t * ql1 = x[i].qs + q_offset;- __global const uint8_t * ql2 = ql1 + 64;- __global const uint8_t * qh = x[i].qh + l0;- __global const float * y1 = yy + i*QK_K + y_offset;- __global const float * y2 = y1 + 128;-- const float dall = vload_half(0, &x[i].d);- const float dmin = vload_half(0, &x[i].dmin);-- __global const uint16_t * a = (__global const uint16_t *)x[i].scales;- aux[0] = a[im+0] & kmask1;- aux[1] = a[im+2] & kmask1;- aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);- aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);-- float4 sum = (float4)(0.f);- float smin = 0;- for (int l = 0; l < n; ++l) {- sum.x += y1[l+ 0] * ((ql1[l+ 0] & 0xF) + (qh[l+ 0] & (hm1 << 0) ? 16 : 0))- + y1[l+16] * ((ql1[l+16] & 0xF) + (qh[l+16] & (hm1 << 0) ? 16 : 0));- sum.y += y1[l+32] * ((ql1[l+ 0] >> 4) + (qh[l+ 0] & (hm1 << 1) ? 16 : 0))- + y1[l+48] * ((ql1[l+16] >> 4) + (qh[l+16] & (hm1 << 1) ? 16 : 0));- sum.z += y2[l+ 0] * ((ql2[l+ 0] & 0xF) + (qh[l+ 0] & (hm2 << 0) ? 16 : 0))- + y2[l+16] * ((ql2[l+16] & 0xF) + (qh[l+16] & (hm2 << 0) ? 16 : 0));- sum.w += y2[l+32] * ((ql2[l+ 0] >> 4) + (qh[l+ 0] & (hm2 << 1) ? 16 : 0))- + y2[l+48] * ((ql2[l+16] >> 4) + (qh[l+16] & (hm2 << 1) ? 16 : 0));- smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]- + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];- }- tmp[16 * ix + tid] += dall * (sum.x * sc[0] + sum.y * sc[1] + sum.z * sc[4] + sum.w * sc[5]) - dmin * smin;-- }-- // sum up partial sums and write back result- barrier(CLK_LOCAL_MEM_FENCE);- for (int s=16; s>0; s>>=1) {- if (tid < s) {- tmp[tid] += tmp[tid + s];- }- barrier(CLK_LOCAL_MEM_FENCE);- }- if (tid == 0) {- dst[row] = tmp[0];- }-}--__kernel void dequantize_mul_mat_vec_q6_K(__global const struct block_q6_K * xx, __local float* tmp, __global const float * yy, __global float * dst, const int ncols) {-- const int row = get_group_id(0);-- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row + get_global_offset(0);-- __global const struct block_q6_K * x = xx + ib0;-- const int tid = get_local_id(0)/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = get_local_id(0)%K_QUANTS_PER_ITERATION; // 0 or 0, 1-- const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8-- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0...15 or 0...7--\n#if K_QUANTS_PER_ITERATION == 1\n- const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15- const int is = 0;--\n#else\n-- const int l0 = 4 * in; // 0, 4, 8, ..., 28- const int is = in / 4;--\n#endif\n-- const int ql_offset = 64*im + l0;- const int qh_offset = 32*im + l0;- const int s_offset = 8*im + is;- const int y_offset = 128*im + l0;-- tmp[16 * ix + tid] = 0; // partial sum for thread in warp-- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {-- __global const float * y = yy + i * QK_K + y_offset;- __global const uint8_t * ql = x[i].ql + ql_offset;- __global const uint8_t * qh = x[i].qh + qh_offset;- __global const int8_t * s = x[i].scales + s_offset;-- const float d = vload_half(0, &x[i].d);--\n#if K_QUANTS_PER_ITERATION == 1\n- float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)- + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)- + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)- + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)- + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)- + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)- + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)- +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);- tmp[16 * ix + tid] += sum;-\n#else\n- float sum = 0;- for (int l = 0; l < 4; ++l) {- sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)- + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)- + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)- + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);- }- tmp[16 * ix + tid] += sum;-\n#endif\n-- }-- // sum up partial sums and write back result- barrier(CLK_LOCAL_MEM_FENCE);- for (int s=16; s>0; s>>=1) {- if (tid < s) {- tmp[tid] += tmp[tid + s];- }- barrier(CLK_LOCAL_MEM_FENCE);- }- if (tid == 0) {- dst[row] = tmp[0];- }-}--);---std::string dequant_template = MULTILINE_QUOTE(-__kernel void KERNEL_NAME(__global X_TYPE* x, __global float* y) {- const int i = get_group_id(0)*get_local_size(0) + get_local_id(0)*2;-- if (i >= get_global_size(0)) {- return;- }-- const uint qk = QUANT_K;- const uint qr = QUANT_R;-- const int ib = i/qk + get_global_offset(0); // block index- const int iqs = (i%qk)/qr; // quant index- const int iybs = i - i%qk; // y block start index- const int y_offset = qr == 1 ? 1 : qk/2;-- // dequantize- float v0, v1;- DEQUANT_FUNC(x, ib, iqs, &v0, &v1);- y[iybs + iqs + 0] = v0;- y[iybs + iqs + y_offset] = v1;-}-);--std::string dequant_mul_mat_vec_template = MULTILINE_QUOTE(-__kernel void KERNEL_NAME(__global X_TYPE* x, __local float* tmp, __global float* y, __global float* dst, const int ncols) {- const int local_size = get_local_size(0);- const int row = get_group_id(0);- const int tid = get_local_id(0);-- const uint qk = QUANT_K;- const uint qr = QUANT_R;-- const int col_step = local_size * 2;- const int y_offset = qr == 1 ? 1 : qk/2;-- x += get_global_offset(0);-- tmp[tid] = 0;-- for (int col = tid*2; col < ncols; col += col_step) {- const int ib = (row*ncols + col)/qk; // block index- const int iqs = (col%qk)/qr; // quant index- const int iybs = col - col%qk; // y block start index-- // dequantize- float v0, v1;- DEQUANT_FUNC(x, ib, iqs, &v0, &v1);-- // matrix multiplication- tmp[tid] += v0 * y[iybs + iqs + 0];- tmp[tid] += v1 * y[iybs + iqs + y_offset];- }-- // sum up partial sums and write back result- barrier(CLK_LOCAL_MEM_FENCE);- for (int s=local_size/2; s>0; s>>=1) {- if (tid < s) {- tmp[tid] += tmp[tid + s];- }- barrier(CLK_LOCAL_MEM_FENCE);- }- if (tid == 0) {- dst[row] = tmp[0];- }-}-);---std::string mul_template = MULTILINE_QUOTE(-__kernel void KERNEL_NAME(__global TYPE* x, const int x_offset, __global TYPE* y, const int y_offset, __global TYPE* dst, const int dst_offset, const int ky) {- const int i = get_group_id(0)*get_local_size(0) + get_local_id(0);-- if (i >= get_global_size(0)) {- return;- }-- dst[dst_offset + i] = x[x_offset + i] * y[y_offset + i%ky];-}-);--#define CL_CHECK(err) \- do { \- cl_int err_ = (err); \- if (err_ != CL_SUCCESS) { \- fprintf(stderr, "ggml_opencl: %s error %d at %s:%d\n", \- #err, err_, __FILE__, __LINE__); \- exit(1); \- } \- } while (0)--#define CLBLAST_CHECK(err) \- do { \- CLBlastStatusCode err_ = (err); \- if (err_ != CLBlastSuccess) { \- fprintf(stderr, "ggml_opencl: %s error %d at %s:%d\n", \- #err, err_, __FILE__, __LINE__); \- exit(1); \- } \- } while (0)--std::array<std::string, 5> dequant_str_keys = {- "KERNEL_NAME", "X_TYPE", "QUANT_K", "QUANT_R", "DEQUANT_FUNC"-};--std::array<std::string, 30> dequant_str_values = {- "dequantize_row_q4_0", "struct block_q4_0", "QK4_0", "QR4_0", "dequantize_q4_0",- "dequantize_row_q4_1", "struct block_q4_1", "QK4_1", "QR4_1", "dequantize_q4_1",- "dequantize_row_q5_0", "struct block_q5_0", "QK5_0", "QR5_0", "dequantize_q5_0",- "dequantize_row_q5_1", "struct block_q5_1", "QK5_1", "QR5_1", "dequantize_q5_1",- "dequantize_row_q8_0", "struct block_q8_0", "QK8_0", "QR8_0", "dequantize_q8_0",- "convert_row_f16", "half", "1", "1", "convert_f16"-};--std::array<std::string, 30> dequant_mul_mat_vec_str_values = {- "dequantize_mul_mat_vec_q4_0", "struct block_q4_0", "QK4_0", "QR4_0", "dequantize_q4_0",- "dequantize_mul_mat_vec_q4_1", "struct block_q4_1", "QK4_1", "QR4_1", "dequantize_q4_1",- "dequantize_mul_mat_vec_q5_0", "struct block_q5_0", "QK5_0", "QR5_0", "dequantize_q5_0",- "dequantize_mul_mat_vec_q5_1", "struct block_q5_1", "QK5_1", "QR5_1", "dequantize_q5_1",- "dequantize_mul_mat_vec_q8_0", "struct block_q8_0", "QK8_0", "QR8_0", "dequantize_q8_0",- "convert_mul_mat_vec_f16", "half", "1", "1", "convert_f16"-};--std::array<std::string, 2> mul_str_keys = {- "KERNEL_NAME", "TYPE"-};-std::array<std::string, 2> mul_str_values = {- "mul_f32", "float"-};--static std::string& replace(std::string& s, const std::string& from, const std::string& to) {- size_t pos = 0;- while ((pos = s.find(from, pos)) != std::string::npos) {- s.replace(pos, from.length(), to);- pos += to.length();- }- return s;-}--static std::string generate_kernels() {- std::stringstream src;- src << program_source << '\n';- src << k_quants_source << '\n';- for (size_t i = 0; i < dequant_str_values.size(); i += dequant_str_keys.size()) {- std::string dequant_kernel = dequant_template;- std::string dmmv_kernel = dequant_mul_mat_vec_template;- for (size_t j = 0; j < dequant_str_keys.size(); j++) {- replace(dequant_kernel, dequant_str_keys[j], dequant_str_values[i + j]);- replace(dmmv_kernel, dequant_str_keys[j], dequant_mul_mat_vec_str_values[i + j]);- }- src << dequant_kernel << '\n';- src << dmmv_kernel << '\n';- }- for (size_t i = 0; i < mul_str_values.size(); i += mul_str_keys.size()) {- std::string mul_kernel = mul_template;- for (size_t j = 0; j < mul_str_keys.size(); j++) {- replace(mul_kernel, mul_str_keys[j], mul_str_values[i + j]);- }- src << mul_kernel << '\n';- }-- return src.str();-}--static cl_platform_id platform;-static cl_device_id device;-static cl_context context;-static cl_command_queue queue;-static cl_program program;-static cl_kernel convert_row_f16_cl;-static cl_kernel dequantize_row_q4_0_cl, dequantize_row_q4_1_cl, dequantize_row_q5_0_cl, dequantize_row_q5_1_cl, dequantize_row_q8_0_cl;-static cl_kernel dequantize_mul_mat_vec_q4_0_cl, dequantize_mul_mat_vec_q4_1_cl, dequantize_mul_mat_vec_q5_0_cl, dequantize_mul_mat_vec_q5_1_cl, dequantize_mul_mat_vec_q8_0_cl, convert_mul_mat_vec_f16_cl;-static cl_kernel dequantize_block_q2_k_cl, dequantize_block_q3_k_cl, dequantize_block_q4_k_cl, dequantize_block_q5_k_cl, dequantize_block_q6_k_cl;-static cl_kernel dequantize_mul_mat_vec_q2_K_cl, dequantize_mul_mat_vec_q3_K_cl, dequantize_mul_mat_vec_q4_K_cl, dequantize_mul_mat_vec_q5_K_cl, dequantize_mul_mat_vec_q6_K_cl;-static cl_kernel mul_f32_cl;-static bool fp16_support;--static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, const char* program_buffer) {- cl_program p;- char *program_log;- size_t program_size;- size_t log_size;- int err;-- program_size = strlen(program_buffer);-- p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err);- if(err < 0) {- fprintf(stderr, "OpenCL error creating program");- exit(1);- }-- std::string compile_opts = "-cl-mad-enable -cl-unsafe-math-optimizations -cl-finite-math-only -cl-fast-relaxed-math "- "-DQK4_0=32 -DQR4_0=2 -DQK4_1=32 -DQR4_1=2 -DQK5_0=32 -DQR5_0=2 -DQK5_1=32 -DQR5_1=2 -DQK8_0=32 -DQR8_0=1 "- "-DQK_K=256 -DK_QUANTS_PER_ITERATION=" + std::to_string(K_QUANTS_PER_ITERATION);-- err = clBuildProgram(p, 0, NULL, compile_opts.c_str(), NULL, NULL);- if(err < 0) {-- clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);- program_log = (char*) malloc(log_size + 1);- program_log[log_size] = '\0';- clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL);- fprintf(stderr, "ggml_opencl: kernel compile error:\n\n%s\n", program_log);- free(program_log);- exit(1);- }-- return p;-}--void ggml_cl_init(void) {- cl_int err;-- struct cl_device;- struct cl_platform {- cl_platform_id id;- unsigned number;- char name[128];- char vendor[128];- struct cl_device * devices;- unsigned n_devices;- struct cl_device * default_device;- };-- struct cl_device {- struct cl_platform * platform;- cl_device_id id;- unsigned number;- cl_device_type type;- char name[128];- };-- enum { NPLAT = 16, NDEV = 16 };-- struct cl_platform platforms[NPLAT];- unsigned n_platforms = 0;- struct cl_device devices[NDEV];- unsigned n_devices = 0;- struct cl_device * default_device = NULL;-- platform = NULL;- device = NULL;-- cl_platform_id platform_ids[NPLAT];- CL_CHECK(clGetPlatformIDs(NPLAT, platform_ids, &n_platforms));-- for (unsigned i = 0; i < n_platforms; i++) {- struct cl_platform * p = &platforms[i];- p->number = i;- p->id = platform_ids[i];- CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL));- CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL));-- cl_device_id device_ids[NDEV];- cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices);- if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) {- p->n_devices = 0;- } else {- CL_CHECK(clGetDeviceIDsError);- }- p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL;- p->default_device = NULL;-- for (unsigned j = 0; j < p->n_devices; j++) {- struct cl_device * d = &devices[n_devices];- d->number = n_devices++;- d->id = device_ids[j];- d->platform = p;- CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL));- CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL));-- if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) {- p->default_device = d;- }- }-- if (default_device == NULL && p->default_device != NULL) {- default_device = p->default_device;- }- }-- if (n_devices == 0) {- fprintf(stderr, "ggml_opencl: could find any OpenCL devices.\n");- exit(1);- }-- char * user_platform_string = getenv("GGML_OPENCL_PLATFORM");- char * user_device_string = getenv("GGML_OPENCL_DEVICE");- int user_platform_number = -1;- int user_device_number = -1;-- unsigned n;- if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) {- user_platform_number = (int)n;- }- if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) {- user_device_number = (int)n;- }- if (user_platform_number != -1 && user_device_number != -1) {- cl_platform* platform = &platforms[user_platform_number];- if ((unsigned)user_device_number >= platform->n_devices) {- fprintf(stderr, "ggml_opencl: invalid device number %d\n", user_device_number);- exit(1);- }- default_device = &platform->devices[user_device_number];- } else {-- struct cl_device * selected_devices = devices;- unsigned n_selected_devices = n_devices;-- if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) {- for (unsigned i = 0; i < n_platforms; i++) {- struct cl_platform * p = &platforms[i];- if (strstr(p->name, user_platform_string) != NULL ||- strstr(p->vendor, user_platform_string) != NULL) {- user_platform_number = (int)i;- break;- }- }- if (user_platform_number == -1) {- fprintf(stderr, "ggml_opencl: no platform matching '%s' was found.\n", user_platform_string);- exit(1);- }- }- if (user_platform_number != -1) {- struct cl_platform * p = &platforms[user_platform_number];- selected_devices = p->devices;- n_selected_devices = p->n_devices;- default_device = p->default_device;- if (n_selected_devices == 0) {- fprintf(stderr, "ggml_opencl: selected platform '%s' does not have any devices.\n", p->name);- exit(1);- }- }-- if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) {- for (unsigned i = 0; i < n_selected_devices; i++) {- struct cl_device * d = &selected_devices[i];- if (strstr(d->name, user_device_string) != NULL) {- user_device_number = d->number;- break;- }- }- if (user_device_number == -1) {- fprintf(stderr, "ggml_opencl: no device matching '%s' was found.\n", user_device_string);- exit(1);- }- }- if (user_device_number != -1) {- selected_devices = &devices[user_device_number];- n_selected_devices = 1;- default_device = &selected_devices[0];- }-- GGML_ASSERT(n_selected_devices > 0);-- if (default_device == NULL) {- default_device = &selected_devices[0];- }- }-- fprintf(stderr, "ggml_opencl: selecting platform: '%s'\n", default_device->platform->name);- fprintf(stderr, "ggml_opencl: selecting device: '%s'\n", default_device->name);- if (default_device->type != CL_DEVICE_TYPE_GPU) {- fprintf(stderr, "ggml_opencl: warning, not a GPU: '%s'.\n", default_device->name);- }-- platform = default_device->platform->id;- device = default_device->id;-- size_t ext_str_size;- clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size);- char *ext_buffer = (char *)alloca(ext_str_size + 1);- clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer, NULL);- ext_buffer[ext_str_size] = '\0'; // ensure it is null terminated- // Check if ext_buffer contains cl_khr_fp16- fp16_support = strstr(ext_buffer, "cl_khr_fp16") != NULL;- fprintf(stderr, "ggml_opencl: device FP16 support: %s\n", fp16_support ? "true" : "false");-- cl_context_properties properties[] = {- (intptr_t)CL_CONTEXT_PLATFORM, (intptr_t)platform, 0- };-- CL_CHECK((context = clCreateContext(properties, 1, &device, NULL, NULL, &err), err));-- CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err),- (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err :- (queue = clCreateCommandQueue(context, device, 0, &err), err)- )));-- const std::string kernel_src = generate_kernels();-- program = build_program_from_source(context, device, kernel_src.c_str());-- // FP16 to FP32 kernel- CL_CHECK((convert_row_f16_cl = clCreateKernel(program, "convert_row_f16", &err), err));-- // Dequantize kernels- CL_CHECK((dequantize_row_q4_0_cl = clCreateKernel(program, "dequantize_row_q4_0", &err), err));- CL_CHECK((dequantize_row_q4_1_cl = clCreateKernel(program, "dequantize_row_q4_1", &err), err));- CL_CHECK((dequantize_row_q5_0_cl = clCreateKernel(program, "dequantize_row_q5_0", &err), err));- CL_CHECK((dequantize_row_q5_1_cl = clCreateKernel(program, "dequantize_row_q5_1", &err), err));- CL_CHECK((dequantize_row_q8_0_cl = clCreateKernel(program, "dequantize_row_q8_0", &err), err));- CL_CHECK((dequantize_row_q8_0_cl = clCreateKernel(program, "dequantize_row_q8_0", &err), err));- CL_CHECK((dequantize_block_q2_k_cl = clCreateKernel(program, "dequantize_block_q2_K", &err), err));- CL_CHECK((dequantize_block_q3_k_cl = clCreateKernel(program, "dequantize_block_q3_K", &err), err));- CL_CHECK((dequantize_block_q4_k_cl = clCreateKernel(program, "dequantize_block_q4_K", &err), err));- CL_CHECK((dequantize_block_q5_k_cl = clCreateKernel(program, "dequantize_block_q5_K", &err), err));- CL_CHECK((dequantize_block_q6_k_cl = clCreateKernel(program, "dequantize_block_q6_K", &err), err));-- // dequant mul mat kernel- CL_CHECK((dequantize_mul_mat_vec_q4_0_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q4_0", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q4_1_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q4_1", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q5_0_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q5_0", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q5_1_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q5_1", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q8_0_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q8_0", &err), err));- CL_CHECK((convert_mul_mat_vec_f16_cl = clCreateKernel(program, "convert_mul_mat_vec_f16", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q2_K_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q2_K", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q3_K_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q3_K", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q4_K_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q4_K", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q5_K_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q5_K", &err), err));- CL_CHECK((dequantize_mul_mat_vec_q6_K_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q6_K", &err), err));-- // mul kernel- CL_CHECK((mul_f32_cl = clCreateKernel(program, "mul_f32", &err), err));-}--static cl_kernel* ggml_get_to_fp32_cl(ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- return &dequantize_row_q4_0_cl;- case GGML_TYPE_Q4_1:- return &dequantize_row_q4_1_cl;- case GGML_TYPE_Q5_0:- return &dequantize_row_q5_0_cl;- case GGML_TYPE_Q5_1:- return &dequantize_row_q5_1_cl;- case GGML_TYPE_Q8_0:- return &dequantize_row_q8_0_cl;- case GGML_TYPE_Q2_K:- return &dequantize_block_q2_k_cl;- case GGML_TYPE_Q3_K:- return &dequantize_block_q3_k_cl;- case GGML_TYPE_Q4_K:- return &dequantize_block_q4_k_cl;- case GGML_TYPE_Q5_K:- return &dequantize_block_q5_k_cl;- case GGML_TYPE_Q6_K:- return &dequantize_block_q6_k_cl;- case GGML_TYPE_F16:- return &convert_row_f16_cl;- default:- return nullptr;- }-}--static size_t ggml_cl_global_denom(ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- return 1;- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- return 4;- case GGML_TYPE_Q4_K:- return 8;- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- return 4;- case GGML_TYPE_F16:- default:- return 1;- }-}--static size_t ggml_cl_local_size(ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- return 0;- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- return 64;- case GGML_TYPE_Q4_K:- return 32;- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- return 64;- case GGML_TYPE_F16:- default:- return 0;- }-}--static cl_kernel* ggml_get_dequantize_mul_mat_vec_cl(ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- return &dequantize_mul_mat_vec_q4_0_cl;- case GGML_TYPE_Q4_1:- return &dequantize_mul_mat_vec_q4_1_cl;- case GGML_TYPE_Q5_0:- return &dequantize_mul_mat_vec_q5_0_cl;- case GGML_TYPE_Q5_1:- return &dequantize_mul_mat_vec_q5_1_cl;- case GGML_TYPE_Q8_0:- return &dequantize_mul_mat_vec_q8_0_cl;- case GGML_TYPE_F16:- return &convert_mul_mat_vec_f16_cl;- case GGML_TYPE_Q2_K:- return &dequantize_mul_mat_vec_q2_K_cl;- case GGML_TYPE_Q3_K:- return &dequantize_mul_mat_vec_q3_K_cl;- case GGML_TYPE_Q4_K:- return &dequantize_mul_mat_vec_q4_K_cl;- case GGML_TYPE_Q5_K:- return &dequantize_mul_mat_vec_q5_K_cl;- case GGML_TYPE_Q6_K:- return &dequantize_mul_mat_vec_q6_K_cl;- default:- return nullptr;- }-}--// buffer pool for cl-#define MAX_CL_BUFFERS 256--struct scoped_spin_lock {- std::atomic_flag& lock;- scoped_spin_lock(std::atomic_flag& lock) : lock(lock) {- while (lock.test_and_set(std::memory_order_acquire)) {- ; // spin- }- }- ~scoped_spin_lock() {- lock.clear(std::memory_order_release);- }- scoped_spin_lock(const scoped_spin_lock&) = delete;- scoped_spin_lock& operator=(const scoped_spin_lock&) = delete;-};--struct cl_buffer {- cl_mem mem;- size_t size = 0;-};--static cl_buffer g_cl_buffer_pool[MAX_CL_BUFFERS];-static std::atomic_flag g_cl_pool_lock = ATOMIC_FLAG_INIT;--static cl_mem ggml_cl_pool_malloc(size_t size, size_t * actual_size) {- scoped_spin_lock lock(g_cl_pool_lock);- cl_int err;-- int best_i = -1;- size_t best_size = std::numeric_limits<size_t>::max(); //smallest unused buffer that fits our needs- int worst_i = -1;- size_t worst_size = 0; //largest unused buffer seen so far- for (int i = 0; i < MAX_CL_BUFFERS; ++i) {- cl_buffer &b = g_cl_buffer_pool[i];- if (b.size > 0 && b.size >= size && b.size < best_size)- {- best_i = i;- best_size = b.size;- }- if (b.size > 0 && b.size > worst_size)- {- worst_i = i;- worst_size = b.size;- }- }- if(best_i!=-1) //found the smallest buffer that fits our needs- {- cl_buffer& b = g_cl_buffer_pool[best_i];- cl_mem mem = b.mem;- *actual_size = b.size;- b.size = 0;- return mem;- }- if(worst_i!=-1) //no buffer that fits our needs, resize largest one to save memory- {- cl_buffer& b = g_cl_buffer_pool[worst_i];- cl_mem mem = b.mem;- b.size = 0;- clReleaseMemObject(mem);- }- cl_mem mem;- CL_CHECK((mem = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err), err));- *actual_size = size;- return mem;-}--static void ggml_cl_pool_free(cl_mem mem, size_t size) {- scoped_spin_lock lock(g_cl_pool_lock);-- for (int i = 0; i < MAX_CL_BUFFERS; ++i) {- cl_buffer& b = g_cl_buffer_pool[i];- if (b.size == 0) {- b.mem = mem;- b.size = size;- return;- }- }- fprintf(stderr, "WARNING: cl buffer pool full, increase MAX_CL_BUFFERS\n");- clReleaseMemObject(mem);-}--void ggml_cl_free_data(const struct ggml_tensor* tensor) {- if (tensor->backend != GGML_BACKEND_GPU) {- return;- }-- cl_mem mem = (cl_mem)tensor->extra;- clReleaseMemObject(mem);-}--static cl_int ggml_cl_h2d_tensor_2d(cl_command_queue queue, cl_mem dst, size_t offset, const struct ggml_tensor * src, uint64_t i3, uint64_t i2, cl_event* ev) {- cl_int err;- const uint64_t ne0 = src->ne[0];- const uint64_t ne1 = src->ne[1];- const uint64_t nb0 = src->nb[0];- const uint64_t nb1 = src->nb[1];- const uint64_t nb2 = src->nb[2];- const uint64_t nb3 = src->nb[3];- const enum ggml_type type = src->type;- const size_t ts = ggml_type_size(type);- const size_t bs = ggml_blck_size(type);- const uint64_t row_size = ts*ne0/bs;-- const char * x = (const char *) src->data + i2*nb2 + i3*nb3;- if (nb0 == ts && nb1 == row_size) {- return clEnqueueWriteBuffer(queue, dst, CL_FALSE, offset, ne1*row_size, x, 0, NULL, ev);- }- if (nb0 == ts) {- const size_t buffer_origin[3] = { offset, 0, 0 };- const size_t host_origin[3] = { 0, 0, 0 };- const size_t region[3] = { row_size, ne1, 1 };- return clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, row_size, 0, nb1, 0, x, 0, NULL, ev);- }- std::vector<cl_event> events;- if (ev && ne1>1) events.reserve(ne1-1);- for (uint64_t i1 = 0; i1 < ne1; i1++) {- // pretend the row is a matrix with cols=1- const size_t buffer_origin[3] = { offset + i1*row_size, 0, 0 };- const size_t host_origin[3] = { 0, 0, 0 };- const size_t region[3] = { ts, ne0/bs, 1 };- // if an event is requested, make the last write wait for all previous writes to complete- if (ev && i1) {- events.push_back(*ev);- }- cl_uint nevents = i1 == ne1-1 ? events.size() : 0U;- err = clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, ts, 0, nb0, 0, x + i1*nb1, nevents, nevents ? events.data() : nullptr, ev);- if (err != CL_SUCCESS) {- for (auto event : events) {- clReleaseEvent(event);- }- return err;- }- }- for (auto event : events) {- CL_CHECK(clReleaseEvent(event));- }- return CL_SUCCESS;-}--static void ggml_cl_mul_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- GGML_ASSERT(src1->backend == GGML_BACKEND_GPU);- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t ne03 = src0->ne[3];- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];- const int64_t ne13 = src1->ne[3];- const int nb2 = dst->nb[2];- const int nb3 = dst->nb[3];- size_t x_size;- size_t d_size;-- cl_mem d_X = ggml_cl_pool_malloc(ne00 * ne01 * sizeof(float), &x_size); // src0- cl_mem d_Y = (cl_mem) src1->extra; // src1 is already on device, broadcasted.- cl_mem d_D = ggml_cl_pool_malloc(ne00 * ne01 * sizeof(float), &d_size); // dst--- for (int64_t i03 = 0; i03 < ne03; i03++) {- for (int64_t i02 = 0; i02 < ne02; i02++) {- cl_event ev;-- // copy src0 to device- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, &ev));-- const int64_t i13 = i03%ne13;- const int64_t i12 = i02%ne12;- const int i1 = i13*ne12*ne11 + i12*ne11;-- cl_int x_offset = 0;- cl_int y_offset = i1*ne10;- cl_int d_offset = 0;-- size_t global = ne00 * ne01;- cl_int ky = ne10 * ne11;-- CL_CHECK(clSetKernelArg(mul_f32_cl, 0, sizeof(cl_mem), &d_X));- CL_CHECK(clSetKernelArg(mul_f32_cl, 1, sizeof(cl_int), &x_offset));- CL_CHECK(clSetKernelArg(mul_f32_cl, 2, sizeof(cl_mem), &d_Y));- CL_CHECK(clSetKernelArg(mul_f32_cl, 3, sizeof(cl_int), &y_offset));- CL_CHECK(clSetKernelArg(mul_f32_cl, 4, sizeof(cl_mem), &d_D));- CL_CHECK(clSetKernelArg(mul_f32_cl, 5, sizeof(cl_int), &d_offset));- CL_CHECK(clSetKernelArg(mul_f32_cl, 6, sizeof(cl_int), &ky));- CL_CHECK(clEnqueueNDRangeKernel(queue, mul_f32_cl, 1, NULL, &global, NULL, 1, &ev, NULL));-- CL_CHECK(clReleaseEvent(ev));- CL_CHECK(clFinish(queue));-- // copy dst to host- float * d = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);- CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * ne00*ne01, d, 0, NULL, NULL));- }- }- ggml_cl_pool_free(d_X, x_size);- ggml_cl_pool_free(d_D, d_size);-}--void ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {- GGML_ASSERT(src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);- ggml_cl_mul_f32(src0, src1, dst);-}--static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t ne03 = src0->ne[3];-- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];- const int64_t ne13 = src1->ne[3];-- const int nb2 = dst->nb[2];- const int nb3 = dst->nb[3];-- const int64_t r2 = ne12 / ne02;- const int64_t r3 = ne13 / ne03;-- const float alpha = 1.0f;- const float beta = 0.0f;- const int x_ne = ne01 * ne00;- const int y_ne = ne11 * ne10;- const int d_ne = ne11 * ne01;-- size_t x_size;- size_t y_size;- size_t d_size;- cl_mem d_X;- if (src0->backend == GGML_BACKEND_GPU) { // NOLINT- d_X = (cl_mem) src0->extra;- } else {- d_X = ggml_cl_pool_malloc(sizeof(float) * x_ne, &x_size);- }- cl_mem d_Y = ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size);- cl_mem d_D = ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size);-- size_t x_offset = 0;-- for (int64_t i03 = 0; i03 < ne03; i03++) {- // TODO: copy src0 here when r3>1- for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) {- for (int64_t i02 = 0; i02 < ne02; i02++) {- if (src0->backend == GGML_BACKEND_GPU) {- x_offset = (i03 * ne02 + i02) * x_ne;- } else {- // copy src0 to device- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));- }-- for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) {- // copy src1 to device- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, NULL));-- CL_CHECK(clFinish(queue));-- // compute- cl_event ev_sgemm;- clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,- clblast::Transpose::kYes, clblast::Transpose::kNo,- ne01, ne11, ne10,- alpha,- d_X, x_offset, ne00,- d_Y, 0, ne10,- beta,- d_D, 0, ne01,- &queue, &ev_sgemm);-- if (status != clblast::StatusCode::kSuccess) {- GGML_ASSERT(false);- }-- // copy dst to host- float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);- CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL));- }- }- }- }-- if (src0->backend != GGML_BACKEND_GPU) {- ggml_cl_pool_free(d_X, x_size);- }- ggml_cl_pool_free(d_Y, y_size);- ggml_cl_pool_free(d_D, d_size);-}--static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, void * wdata, size_t wsize) {- GGML_ASSERT(fp16_support);-- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t ne03 = src0->ne[3];-- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];- const int64_t ne13 = src1->ne[3];-- const int nb10 = src1->nb[0];- const int nb11 = src1->nb[1];- const int nb12 = src1->nb[2];- const int nb13 = src1->nb[3];-- const int nb2 = dst->nb[2];- const int nb3 = dst->nb[3];-- const int64_t r2 = ne12 / ne02;- const int64_t r3 = ne13 / ne03;-- const ggml_fp16_t alpha = ggml_fp32_to_fp16(1.0f);- const ggml_fp16_t beta = ggml_fp32_to_fp16(0.0f);- const int x_ne = ne01 * ne00;- const int y_ne = ne11 * ne10;- const int d_ne = ne11 * ne01;-- GGML_ASSERT(wsize >= sizeof(ggml_fp16_t) * y_ne);- GGML_ASSERT(wsize >= sizeof(ggml_fp16_t) * d_ne);- ggml_fp16_t * const tmp = (ggml_fp16_t *) wdata;-- size_t x_size;- size_t y_size;- size_t d_size;- cl_mem d_X;- if (src0->backend == GGML_BACKEND_GPU) { // NOLINT- d_X = (cl_mem) src0->extra;- } else {- d_X = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * x_ne, &x_size);- }- cl_mem d_Y = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * y_ne, &y_size);- cl_mem d_D = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * d_ne, &d_size);-- bool src1_cont_rows = nb10 == sizeof(float);- bool src1_cont_cols = (size_t)nb11 == ne11*sizeof(float);-- size_t x_offset = 0;-- for (int64_t i03 = 0; i03 < ne03; i03++) {- // TODO: copy src0 here when r3>1- for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) {- for (int64_t i02 = 0; i02 < ne02; i02++) {- if (src0->backend == GGML_BACKEND_GPU) {- x_offset = (i03 * ne02 + i02) * x_ne;- } else {- // copy src0 to device- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));- }-- for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) {- // convert src1 to fp16- // TODO: use multiple threads- char * src1i = (char *) src1->data + i13*nb13 + i12*nb12;- if (src1_cont_rows) {- if (src1_cont_cols) {- ggml_fp32_to_fp16_row((float *) src1i, tmp, ne10*ne11);- }- else {- for (int64_t i11 = 0; i11 < ne11; i11++) {- ggml_fp32_to_fp16_row((float *) (src1i + i11*nb11), tmp + i11*ne10, ne10);- }- }- }- else {- for (int64_t i11 = 0; i11 < ne11; i11++) {- for (int64_t i10 = 0; i10 < ne10; i10++) {- // very slow due to no inlining- tmp[i11*ne10 + i10] = ggml_fp32_to_fp16(*(float *) (src1i + i11*nb11 + i10*nb10));- }- }- }-- // copy src1 to device- CL_CHECK(clEnqueueWriteBuffer(queue, d_Y, false, 0, sizeof(ggml_fp16_t) * y_ne, tmp, 0, NULL, NULL));-- CL_CHECK(clFinish(queue));-- // compute- cl_event ev_sgemm;- clblast::StatusCode status = clblast::Gemm<cl_half>(clblast::Layout::kColMajor,- clblast::Transpose::kYes, clblast::Transpose::kNo,- ne01, ne11, ne10,- alpha,- d_X, x_offset, ne00,- d_Y, 0, ne10,- beta,- d_D, 0, ne01,- &queue, &ev_sgemm);-- if (status != clblast::StatusCode::kSuccess) {- GGML_ASSERT(false);- }-- // copy dst to host, then convert to float- CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(ggml_fp16_t) * d_ne, tmp, 1, &ev_sgemm, NULL));-- float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);-- ggml_fp16_to_fp32_row(tmp, d, d_ne);- }- }- }- }-- if (src0->backend != GGML_BACKEND_GPU) {- ggml_cl_pool_free(d_X, x_size);- }- ggml_cl_pool_free(d_Y, y_size);- ggml_cl_pool_free(d_D, d_size);-}--static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t ne03 = src0->ne[3];-- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];- const int64_t ne13 = src1->ne[3];-- const int nb2 = dst->nb[2];- const int nb3 = dst->nb[3];- const ggml_type type = src0->type;- const bool mul_mat_vec = ne11 == 1 && ne00%2 == 0;-- const int64_t r2 = ne12 / ne02;- const int64_t r3 = ne13 / ne03;-- const float alpha = 1.0f;- const float beta = 0.0f;- const int x_ne = ne01 * ne00;- const int y_ne = ne11 * ne10;- const int d_ne = ne11 * ne01;- const int x_bps = x_ne / ggml_blck_size(type); // blocks per 2D slice- const size_t q_sz = ggml_type_size(type) * x_bps;-- size_t x_size;- size_t y_size;- size_t d_size;- size_t q_size;- cl_mem d_X;- if (!mul_mat_vec) {- d_X = ggml_cl_pool_malloc(sizeof(float) * x_ne, &x_size);- }- cl_mem d_Y = ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size);- cl_mem d_D = ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size);- cl_mem d_Q;- if (src0->backend == GGML_BACKEND_CPU) {- d_Q = ggml_cl_pool_malloc(q_sz, &q_size);- }-- cl_kernel* to_fp32_cl = ggml_get_to_fp32_cl(type);- cl_kernel* dmmv = ggml_get_dequantize_mul_mat_vec_cl(type);- GGML_ASSERT(to_fp32_cl != nullptr);-- const size_t global_denom = ggml_cl_global_denom(type);- const size_t local = mul_mat_vec ? CL_DMMV_LOCAL_SIZE : ggml_cl_local_size(type);-- size_t ev_idx = 0;- std::vector<cl_event> events;-- for (int64_t i03 = 0; i03 < ne03; i03++) {- // TODO: copy and dequantize src0 here when r3>1- for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) {- for (int64_t i02 = 0; i02 < ne02; i02++) {- // copy src0 to device if necessary- if (src0->backend == GGML_BACKEND_CPU) {- events.emplace_back();- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Q, 0, src0, i03, i02, events.data() + ev_idx++));- } else if (src0->backend == GGML_BACKEND_GPU) {- d_Q = (cl_mem) src0->extra;- } else {- GGML_ASSERT(false);- }-- if (!mul_mat_vec) {- // convert src0 to fp32 on device- const size_t global = x_ne / global_denom;- const size_t offset = src0->backend == GGML_BACKEND_GPU ? (i03 * ne02 + i02) * x_bps : 0;- CL_CHECK(clSetKernelArg(*to_fp32_cl, 0, sizeof(cl_mem), &d_Q));- CL_CHECK(clSetKernelArg(*to_fp32_cl, 1, sizeof(cl_mem), &d_X));- CL_CHECK(clEnqueueNDRangeKernel(queue, *to_fp32_cl, 1, &offset, &global, local > 0 ? &local : NULL, events.size(), !events.empty() ? events.data() : NULL, NULL));- }-- for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) {- if (mul_mat_vec) { // specialized dequantize_mul_mat_vec kernel- // copy src1 to device- events.emplace_back();- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, events.data() + ev_idx++));-- // compute- const size_t global = ne01 * local;- const size_t offset = src0->backend == GGML_BACKEND_GPU ? (i03 * ne02 + i02) * x_bps : 0;- const cl_int ncols = ne00;- events.emplace_back();- CL_CHECK(clSetKernelArg(*dmmv, 0, sizeof(cl_mem), &d_Q));- CL_CHECK(clSetKernelArg(*dmmv, 1, sizeof(float) * local, NULL));- CL_CHECK(clSetKernelArg(*dmmv, 2, sizeof(cl_mem), &d_Y));- CL_CHECK(clSetKernelArg(*dmmv, 3, sizeof(cl_mem), &d_D));- CL_CHECK(clSetKernelArg(*dmmv, 4, sizeof(cl_int), &ncols));- CL_CHECK(clEnqueueNDRangeKernel(queue, *dmmv, 1, &offset, &global, &local, events.size() - 1, events.data(), events.data() + ev_idx++));- } else { // CLBlast matrix matrix multiplication- // copy src1 to device- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, NULL));-- // wait for conversion- CL_CHECK(clFinish(queue));-- // compute- events.emplace_back();- clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,- clblast::Transpose::kYes, clblast::Transpose::kNo,- ne01, ne11, ne10,- alpha,- d_X, 0, ne00,- d_Y, 0, ne10,- beta,- d_D, 0, ne01,- &queue, events.data() + ev_idx++);-- if (status != clblast::StatusCode::kSuccess) {- GGML_ASSERT(false);- }- }-- // copy dst to host- float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);- CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &events[events.size() - 1], NULL));- for (auto *event : events) {- clReleaseEvent(event);- }-- ev_idx = 0;- events.clear();- }- }- }- }-- if (!mul_mat_vec) {- ggml_cl_pool_free(d_X, x_size);- }- ggml_cl_pool_free(d_Y, y_size);- ggml_cl_pool_free(d_D, d_size);- if (src0->backend == GGML_BACKEND_CPU) {- ggml_cl_pool_free(d_Q, q_size);- }-}---bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {- const int64_t ne10 = src1->ne[0];-- const int64_t ne0 = dst->ne[0];- const int64_t ne1 = dst->ne[1];-- // TODO: find the optimal values for these- if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) &&- src1->type == GGML_TYPE_F32 &&- dst->type == GGML_TYPE_F32 &&- ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_GPU)) {- return true;- }-- return false;-}--static bool ggml_cl_mul_mat_use_f16(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * /* dst */) {- // If device doesn't support FP16- if (!fp16_support) {- return false;- }-- size_t src0_sz = ggml_nbytes(src0);- size_t src1_sz = ggml_nbytes(src1);-- // mul_mat_q: src0 is converted to fp32 on device- size_t mul_mat_q_transfer = src0_sz + src1_sz;-- // mul_mat_f16: src1 is converted to fp16 on cpu- size_t mul_mat_f16_transfer = src0_sz + sizeof(ggml_fp16_t) * ggml_nelements(src1);-- // choose the smaller one to transfer to the device- // TODO: this is not always the best choice due to the overhead of converting to fp16- return mul_mat_f16_transfer < mul_mat_q_transfer;-}--void ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize) {- GGML_ASSERT(ggml_cl_can_mul_mat(src0, src1, dst));-- if (src0->type == GGML_TYPE_F32) {- ggml_cl_mul_mat_f32(src0, src1, dst);- }- else if (src0->type == GGML_TYPE_F16) {- if (ggml_cl_mul_mat_use_f16(src0, src1, dst)) {- ggml_cl_mul_mat_f16(src0, src1, dst, wdata, wsize);- }- else {- ggml_cl_mul_mat_q_f32(src0, src1, dst);- }- }- else if (ggml_is_quantized(src0->type)) {- ggml_cl_mul_mat_q_f32(src0, src1, dst);- }- else {- GGML_ASSERT(false);- }-}--size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {- if (src0->type == GGML_TYPE_F16 && ggml_cl_mul_mat_use_f16(src0, src1, dst)) {- return sizeof(ggml_fp16_t) * std::max(src1->ne[0] * src1->ne[1], dst->ne[0] * dst->ne[1]);- }- return 0;-}--void ggml_cl_transform_tensor(void * data, ggml_tensor * tensor) {- const int64_t ne0 = tensor->ne[0];- const int64_t ne1 = tensor->ne[1];- const int64_t ne2 = tensor->ne[2];- const int64_t ne3 = tensor->ne[3];-- const ggml_type type = tensor->type;- const size_t s_sz = ggml_type_size(type) * (size_t) (ne0 * ne1 / ggml_blck_size(type));- const size_t q_sz = s_sz * (size_t) (ne2 * ne3);-- size_t q_size;- cl_mem dst = ggml_cl_pool_malloc(q_sz, &q_size);-- tensor->data = data;- // copy tensor to device- size_t offset = 0;- for (int64_t i3 = 0; i3 < ne3; i3++) {- for (int64_t i2 = 0; i2 < ne2; i2++) {- CL_CHECK(ggml_cl_h2d_tensor_2d(queue, dst, offset, tensor, i3, i2, NULL));- offset += s_sz;- }- }-- CL_CHECK(clFinish(queue));-- tensor->extra = dst;- GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);-}diff --git a/llama.cpp/ggml-opencl.h b/llama.cpp/ggml-opencl.hdeleted file mode 100644index a92b445c..00000000--- a/llama.cpp/ggml-opencl.h+++ /dev/null@@ -1,25 +0,0 @@-#pragma once--#include "ggml.h"--#ifdef __cplusplus-extern "C" {-#endif--void ggml_cl_init(void);--void ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);-bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);-size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);-void ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize);--void * ggml_cl_host_malloc(size_t size);-void ggml_cl_host_free(void * ptr);--void ggml_cl_free_data(const struct ggml_tensor* tensor);--void ggml_cl_transform_tensor(void * data, struct ggml_tensor * tensor);--#ifdef __cplusplus-}-#endifdiff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 1d079513..991821d0 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -18637,14 +18637,14 @@ struct gguf_context {void * data;};-static bool gguf_fread_el(struct ggml_file * file, void * dst, size_t size, size_t * offset) {- const size_t n = ggml_file_read(file, dst, size);+static bool gguf_fread_el(struct llamafile * file, void * dst, size_t size, size_t * offset) {+ const size_t n = llamafile_read(file, dst, size);*offset += n;return n == size;}// NOTE: temporary handling of GGUFv1 >> remove after Oct 2023-static bool gguf_fread_str(struct ggml_file * file, struct gguf_str * p, size_t * offset) {+static bool gguf_fread_str(struct llamafile * file, struct gguf_str * p, size_t * offset) {p->n = 0;p->data = NULL;@@ -18676,7 +18676,7 @@ struct gguf_context * gguf_init_empty(void) {return ctx;}-struct gguf_context * gguf_init_from_file(struct ggml_file * file, struct gguf_init_params params) {+struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_init_params params) {// offset from start of filesize_t offset = 0;@@ -18718,7 +18718,7 @@ struct gguf_context * gguf_init_from_file(struct ggml_file * file, struct gguf_iif (ctx->header.version == 1) {fprintf(stderr, "%s: GGUFv1 is no longer supported. please use a more up-to-date version\n", __func__);- ggml_file_close(file);+ llamafile_close(file);gguf_free(ctx);return NULL;}@@ -18844,7 +18844,7 @@ struct gguf_context * gguf_init_from_file(struct ggml_file * file, struct gguf_iif (offset_pad != 0) {offset += ctx->alignment - offset_pad;- ggml_file_seek(file, offset, SEEK_SET);+ llamafile_seek(file, offset, SEEK_SET);}}@@ -19386,12 +19386,12 @@ void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const vo}}-//static void gguf_fwrite_str(struct ggml_file * file, const struct gguf_str * val) {+//static void gguf_fwrite_str(struct llamafile * file, const struct gguf_str * val) {// fwrite(&val->n, sizeof(val->n), 1, file);// fwrite(val->data, sizeof(char), val->n, file);//}//-//static void gguf_fwrite_el(struct ggml_file * file, const void * val, size_t size) {+//static void gguf_fwrite_el(struct llamafile * file, const void * val, size_t size) {// fwrite(val, sizeof(char), size, file);//}@@ -19564,12 +19564,12 @@ static void gguf_write_to_buf(const struct gguf_context * ctx, struct gguf_buf *}}-void gguf_write_to_file(const struct gguf_context * ctx, struct ggml_file * file, bool only_meta) {+void gguf_write_to_file(const struct gguf_context * ctx, struct llamafile * file, bool only_meta) {struct gguf_buf buf = gguf_buf_init(16*1024);gguf_write_to_buf(ctx, &buf, only_meta);- ggml_file_write(file, buf.data, buf.offset);+ llamafile_write(file, buf.data, buf.offset);gguf_buf_free(buf);}diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 16cda397..680c11e3 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -209,6 +209,7 @@#include <stdint.h>#include <stddef.h>#include <stdbool.h>+#include "llamafile.h"#define GGML_FILE_MAGIC 0x67676d6c // "ggml"#define GGML_FILE_VERSION 1@@ -1967,22 +1968,6 @@ extern "C" {GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist);- //- // file- //-- struct ggml_file;- struct ggml_file *ggml_file_open(const char * fname, const char * mode);- void ggml_file_close(struct ggml_file * file);- long ggml_file_read(struct ggml_file * file, void * ptr, size_t len);- long ggml_file_write(struct ggml_file * file, const void * ptr, size_t len);- void ggml_file_seek(struct ggml_file * file, size_t offset, int whence);- void * ggml_file_content(struct ggml_file * file);- size_t ggml_file_tell(struct ggml_file * file);- size_t ggml_file_size(struct ggml_file * file);- size_t ggml_file_size(struct ggml_file * file);- FILE * ggml_file_fp(struct ggml_file * file);-//// gguf//@@ -2014,7 +1999,7 @@ extern "C" {};GGML_API struct gguf_context * gguf_init_empty(void);- GGML_API struct gguf_context * gguf_init_from_file(struct ggml_file * file, struct gguf_init_params params);+ GGML_API struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_init_params params);//GGML_API struct gguf_context * gguf_init_from_buffer(..);GGML_API void gguf_free(struct gguf_context * ctx);@@ -2098,7 +2083,7 @@ extern "C" {//// write the entire context to a binary file- GGML_API void gguf_write_to_file(const struct gguf_context * ctx, struct ggml_file * file, bool only_meta);+ GGML_API void gguf_write_to_file(const struct gguf_context * ctx, struct llamafile * file, bool only_meta);// get the size in bytes of the meta data (header, kv pairs, tensor info) including paddingGGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);@@ -2156,12 +2141,6 @@ extern "C" {GGML_API ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);- GGML_API void ggml_check_cpu(void);- GGML_API void ggml_get_app_dir(char *path, size_t size);- GGML_API void ggml_schlep(const void *data, size_t size);- GGML_API bool ggml_extract(const char *zip, const char *to);- GGML_API int ggml_is_file_newer_than(const char *path, const char *other);-#ifdef __cplusplus}#endifdiff --git a/llama.cpp/grammar-parser.cc b/llama.cpp/grammar-parser.cppsimilarity index 100%rename from llama.cpp/grammar-parser.ccrename to llama.cpp/grammar-parser.cppdiff --git a/llama.cpp/llama.cc b/llama.cpp/llama.cppsimilarity index 99%rename from llama.cpp/llama.ccrename to llama.cpp/llama.cppindex e27713b9..33e6e865 100644--- a/llama.cpp/llama.cc+++ b/llama.cpp/llama.cpp@@ -1,7 +1,6 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-// vi: set net ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#define _COSMO_SOURCE#include <cosmo.h>#define LLAMA_API_INTERNAL@@ -687,28 +686,28 @@ struct llama_buffer {};struct llama_file {- struct ggml_file * file;+ struct llamafile * file;llama_file(const char * fname, const char * mode) {- file = ggml_file_open(fname, mode);+ file = llamafile_open(fname, mode);if (!file) {ThrowRuntimeError(format("failed to open %s: %s", fname, strerror(errno)));}}size_t tell() const {- return ggml_file_tell(file);+ return llamafile_tell(file);}void seek(size_t offset, int whence) {- return ggml_file_seek(file, offset, whence);+ return llamafile_seek(file, offset, whence);}void read_raw(void * ptr, size_t len) const {if (len == 0) {return;}- long rc = ggml_file_read(file, ptr, len);+ long rc = llamafile_read(file, ptr, len);if (rc == -1) {ThrowRuntimeError(format("read error: %s", strerror(errno)));}@@ -725,7 +724,7 @@ struct llama_file {void write_raw(const void * ptr, size_t len) const {errno = 0;- long rc = ggml_file_write(file, ptr, len);+ long rc = llamafile_write(file, ptr, len);if (rc == -1) {ThrowRuntimeError(format("write error: %s", strerror(errno)));}@@ -736,7 +735,7 @@ struct llama_file {}~llama_file() {- ggml_file_close(file);+ llamafile_close(file);}};@@ -751,17 +750,17 @@ struct llama_mmap {static constexpr bool SUPPORTED = true;llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {- size = ggml_file_size(file->file);- if (!ggml_file_fp(file->file)) {+ size = llamafile_size(file->file);+ if (!llamafile_fp(file->file)) {// file is an uncompressed zip asset// therefore it's already mappedis_owned = false;- addr = ggml_file_content(file->file);- ggml_schlep(addr, size);+ addr = llamafile_content(file->file);+ llamafile_schlep(addr, size);return;}is_owned = true;- int fd = fileno(ggml_file_fp(file->file));+ int fd = fileno(llamafile_fp(file->file));int flags = MAP_SHARED;// prefetch/readahead impairs performance on NUMA systemsif (numa) { prefetch = 0; }@@ -790,7 +789,7 @@ struct llama_mmap {}// report terminal progress of loading weights- ggml_schlep(addr, size);+ llamafile_schlep(addr, size);}~llama_mmap() {@@ -1692,7 +1691,7 @@ struct llama_model_loader {if (!ctx_gguf) {ThrowRuntimeError(format("%s: failed to load model from %s\n", __func__, fname.c_str()));}- ggml_file_seek(file.file, 0, SEEK_SET);+ llamafile_seek(file.file, 0, SEEK_SET);n_kv = gguf_get_n_kv(ctx_gguf);n_tensors = gguf_get_n_tensors(ctx_gguf);@@ -8768,7 +8767,7 @@ static bool llama_load_session_file_internal(struct llama_context * ctx, const c// restore the context state{- const size_t n_state_size_cur = ggml_file_size(file.file) - file.tell();+ const size_t n_state_size_cur = llamafile_size(file.file) - file.tell();const size_t n_state_size_max = llama_get_state_size(ctx);if (n_state_size_cur > n_state_size_max) {diff --git a/llama.cpp/llamafile.h b/llama.cpp/llamafile.hnew file mode 100644index 00000000..bcf7a24e--- /dev/null+++ b/llama.cpp/llamafile.h@@ -0,0 +1,3 @@+#pragma once+// stub imports to keep mkdeps/cuda/metal happy and avoid -iquote flags+#include "llamafile/llamafile.h"diff --git a/llama.cpp/llava/BUILD.mk b/llama.cpp/llava/BUILD.mkindex d453aff4..1126e502 100644--- a/llama.cpp/llava/BUILD.mk+++ b/llama.cpp/llava/BUILD.mk@@ -5,7 +5,7 @@ PKGS += LLAMA_CPP_LLAVALLAMA_CPP_LLAVA_FILES := $(wildcard llama.cpp/llava/*)LLAMA_CPP_LLAVA_HDRS = $(filter %.h,$(LLAMA_CPP_LLAVA_FILES))-LLAMA_CPP_LLAVA_SRCS = $(filter %.cc,$(LLAMA_CPP_LLAVA_FILES))+LLAMA_CPP_LLAVA_SRCS = $(filter %.cpp,$(LLAMA_CPP_LLAVA_FILES)).PHONY: tool/args/args.hdiff --git a/llama.cpp/llava/clip.cc b/llama.cpp/llava/clip.cppsimilarity index 99%rename from llama.cpp/llava/clip.ccrename to llama.cpp/llava/clip.cppindex 8c73373d..741c91e6 100644--- a/llama.cpp/llava/clip.cc+++ b/llama.cpp/llava/clip.cpp@@ -461,11 +461,11 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {/*.ctx = */ &meta,};- struct ggml_file * file;+ struct llamafile * file;struct gguf_context * ctx;- file = ggml_file_open(fname, "rbe");+ file = llamafile_open(fname, "rbe");if (file) ctx = gguf_init_from_file(file, params);- if (file) ggml_file_close(file);+ if (file) llamafile_close(file);if (!file || !ctx) {OpenFailed:ThrowRuntimeError(format("%s: failed to load CLIP model from %s. Does this file exist?\n", __func__, fname));@@ -571,7 +571,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {return nullptr;}- struct ggml_file * fin = ggml_file_open(fname, "rbe");+ struct llamafile * fin = llamafile_open(fname, "rbe");if (!fin) {printf("cannot open model file for loading tensors\n");clip_free(new_clip);@@ -586,17 +586,17 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {ggml_set_name(cur, name);const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i);- ggml_file_seek(fin, offset, SEEK_SET);+ llamafile_seek(fin, offset, SEEK_SET);if (!fin) {printf("%s: failed to seek for tensor %s\n", __func__, name);clip_free(new_clip);return nullptr;}- ggml_file_read(fin, reinterpret_cast<void *>(cur->data), ggml_nbytes(t));+ llamafile_read(fin, reinterpret_cast<void *>(cur->data), ggml_nbytes(t));}- ggml_file_close(fin);+ llamafile_close(fin);}// vision modeldiff --git a/llama.cpp/llava/llava-cli.cc b/llama.cpp/llava/llava-cli.cppsimilarity index 99%rename from llama.cpp/llava/llava-cli.ccrename to llama.cpp/llava/llava-cli.cppindex 1e9944a0..8cd2989a 100644--- a/llama.cpp/llava/llava-cli.cc+++ b/llama.cpp/llava/llava-cli.cpp@@ -280,7 +280,7 @@ static void llava_free(struct llava_context * ctx_llava) {}int main(int argc, char ** argv) {- ggml_check_cpu();+ llamafile_check_cpu();ggml_time_init();gpt_params params;diff --git a/llama.cpp/llava/llava-quantize.cpp b/llama.cpp/llava/llava-quantize.cppnew file mode 100644index 00000000..022ef68d--- /dev/null+++ b/llama.cpp/llava/llava-quantize.cpp@@ -0,0 +1,17 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include <stdio.h>+#include <stdlib.h>+#include "llama.cpp/ggml.h"+#include "clip.h"++int main(int argc, char *argv[]) {+ llamafile_check_cpu();+ if (argc != 4) {+ fprintf(stderr, "Usage: %s INPUT OUTPUT FORMAT\n");+ exit(1);+ }+ if (!clip_model_quantize(argv[1], argv[2], atoi(argv[3]))) {+ exit(1);+ }+}diff --git a/llama.cpp/llava/llava.cc b/llama.cpp/llava/llava.cppsimilarity index 100%rename from llama.cpp/llava/llava.ccrename to llama.cpp/llava/llava.cppdiff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mkindex d3d2ab2c..20e9851c 100644--- a/llama.cpp/main/BUILD.mk+++ b/llama.cpp/main/BUILD.mk@@ -5,8 +5,8 @@ PKGS += LLAMA_CPP_MAINLLAMA_CPP_MAIN_FILES := $(wildcard llama.cpp/main/*)LLAMA_CPP_MAIN_HDRS = $(filter %.h,$(LLAMA_CPP_MAIN_FILES))-LLAMA_CPP_MAIN_SRCS = $(filter %.cc,$(LLAMA_CPP_MAIN_FILES))-LLAMA_CPP_MAIN_OBJS = $(LLAMA_CPP_MAIN_SRCS:%.cc=o/$(MODE)/%.o)+LLAMA_CPP_MAIN_SRCS = $(filter %.cpp,$(LLAMA_CPP_MAIN_FILES))+LLAMA_CPP_MAIN_OBJS = $(LLAMA_CPP_MAIN_SRCS:%.cpp=o/$(MODE)/%.o).PHONY: o/$(MODE)/llama.cpp/maino/$(MODE)/llama.cpp/main: \diff --git a/llama.cpp/main/main.cc b/llama.cpp/main/main.ccindex 15282899..a1407f63 100644--- a/llama.cpp/main/main.cc+++ b/llama.cpp/main/main.cc@@ -1,7 +1,6 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set net ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#define _COSMO_SOURCE#include <cosmo.h>#include "tool/args/args.h"@@ -104,7 +103,7 @@ static int Eval(struct llama_context * ctx, struct llama_batch batch) {}int main(int argc, char ** argv) {- ggml_check_cpu();+ llamafile_check_cpu();ShowCrashReports();LoadZipArgs(&argc, &argv);diff --git a/llama.cpp/perplexity/perplexity.cc b/llama.cpp/perplexity/perplexity.ccindex 4e519bb3..f5c0a609 100644--- a/llama.cpp/perplexity/perplexity.cc+++ b/llama.cpp/perplexity/perplexity.cc@@ -682,7 +682,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {int main(int argc, char ** argv) {gpt_params params;- ggml_check_cpu();+ llamafile_check_cpu();params.n_batch = 512;if (!gpt_params_parse(argc, argv, params)) {diff --git a/llama.cpp/quantize/BUILD.mk b/llama.cpp/quantize/BUILD.mkindex 0bed524f..1af84c73 100644--- a/llama.cpp/quantize/BUILD.mk+++ b/llama.cpp/quantize/BUILD.mk@@ -5,13 +5,13 @@ PKGS += LLAMA_CPP_QUANTIZELLAMA_CPP_QUANTIZE_FILES := $(wildcard llama.cpp/quantize/*)LLAMA_CPP_QUANTIZE_HDRS = $(filter %.h,$(LLAMA_CPP_QUANTIZE_FILES))-LLAMA_CPP_QUANTIZE_SRCS = $(filter %.cc,$(LLAMA_CPP_QUANTIZE_FILES))-LLAMA_CPP_QUANTIZE_OBJS = $(LLAMA_CPP_QUANTIZE_SRCS:%.cc=o/$(MODE)/%.o)+LLAMA_CPP_QUANTIZE_SRCS = $(filter %.cpp,$(LLAMA_CPP_QUANTIZE_FILES))+LLAMA_CPP_QUANTIZE_OBJS = $(LLAMA_CPP_QUANTIZE_SRCS:%.cpp=o/$(MODE)/%.o).PHONY: o/$(MODE)/llama.cpp/quantize-o/$(MODE)/llama.cpp/quantize: \+o/$(MODE)/llama.cpp/quantize: \o/$(MODE)/llama.cpp/quantize/quantize-o/$(MODE)/llama.cpp/quantize/quantize: \- o/$(MODE)/llama.cpp/quantize/quantize.o \+o/$(MODE)/llama.cpp/quantize/quantize: \+ o/$(MODE)/llama.cpp/quantize/quantize.o \o/$(MODE)/llama.cpp/llama.cpp.adiff --git a/llama.cpp/quantize/quantize.cc b/llama.cpp/quantize/quantize.cppsimilarity index 99%rename from llama.cpp/quantize/quantize.ccrename to llama.cpp/quantize/quantize.cppindex 9e174d5e..fce263b2 100644--- a/llama.cpp/quantize/quantize.cc+++ b/llama.cpp/quantize/quantize.cpp@@ -94,7 +94,7 @@ static void usage(const char * executable) {}int main(int argc, char ** argv) {- ggml_check_cpu();+ llamafile_check_cpu();if (argc < 3) {usage(argv[0]);diff --git a/llama.cpp/sampling.cc b/llama.cpp/sampling.cppsimilarity index 100%rename from llama.cpp/sampling.ccrename to llama.cpp/sampling.cppdiff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mkindex 033edc63..50f213cf 100644--- a/llama.cpp/server/BUILD.mk+++ b/llama.cpp/server/BUILD.mk@@ -5,7 +5,7 @@ PKGS += LLAMA_CPP_SERVERLLAMA_CPP_SERVER_FILES := $(wildcard llama.cpp/server/*)LLAMA_CPP_SERVER_HDRS = $(filter %.h,$(LLAMA_CPP_SERVER_FILES))-LLAMA_CPP_SERVER_SRCS = $(filter %.cc,$(LLAMA_CPP_SERVER_FILES))+LLAMA_CPP_SERVER_SRCS = $(filter %.cpp,$(LLAMA_CPP_SERVER_FILES))LLAMA_CPP_SERVER_ASSETS := \$(wildcard llama.cpp/server/public/*) \diff --git a/llama.cpp/server/server.cc b/llama.cpp/server/server.cppsimilarity index 99%rename from llama.cpp/server/server.ccrename to llama.cpp/server/server.cppindex ed167da0..8c49116d 100644--- a/llama.cpp/server/server.cc+++ b/llama.cpp/server/server.cpp@@ -2256,7 +2256,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_conint main(int argc, char **argv){- ggml_check_cpu();+ llamafile_check_cpu();#ifdef __COSMOPOLITAN__LoadZipArgs(&argc, &argv);diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mknew file mode 100644index 00000000..a9ed9739--- /dev/null+++ b/llamafile/BUILD.mk@@ -0,0 +1,15 @@+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐+#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘++PKGS += LLAMAFILE++LLAMAFILE_FILES := $(wildcard llamafile/*.*)+LLAMAFILE_HDRS = $(filter %.h,$(LLAMAFILE_FILES))+LLAMAFILE_SRCS = $(filter %.c,$(LLAMAFILE_FILES))++LLAMAFILE_OBJS = \+ $(LLAMAFILE_SRCS:%.c=o/$(MODE)/%.o) \+ $(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)++.PHONY: o/$(MODE)/llamafile+o/$(MODE)/llamafile: $(LLAMAFILE_OBJS)diff --git a/llama.cpp/cpucheck.c b/llamafile/check_cpu.csimilarity index 97%rename from llama.cpp/cpucheck.crename to llamafile/check_cpu.cindex 6bdd392a..d0b8cd36 100644--- a/llama.cpp/cpucheck.c+++ b/llamafile/check_cpu.c@@ -15,10 +15,9 @@// See the License for the specific language governing permissions and// limitations under the License.-#define _COSMO_SOURCE+#include <errno.h>#include <cosmo.h>#include <stdlib.h>-#include <errno.h>static int on_missing_feature(const char *name) {tinyprint(2, program_invocation_name, ": fatal error: the cpu feature ", name,@@ -39,7 +38,7 @@ static int on_missing_feature(const char *name) {* written to use runtime dispatching should be configured so that* microarchitecture flags only get passed to that specific object*/-void ggml_check_cpu(void) {+void llamafile_check_cpu(void) {if (X86_NEED(SSE3) && !X86_CHECK(SSE3)) {on_missing_feature("SSE3");}diff --git a/llamafile/compcat.cu b/llamafile/compcat.cunew file mode 100644index 00000000..5fe88ad3--- /dev/null+++ b/llamafile/compcat.cu@@ -0,0 +1,30 @@+// https://stackoverflow.com/a/40695640/1653720+#include <cstdio>+#include <cstdlib>+#include <cuda_runtime_api.h>++int main(int argc, char *argv[]) {+ cudaDeviceProp prop;+ cudaError_t status;+ int device_count;+ int device_index = 0;+ if (argc > 1) {+ device_index = atoi(argv[1]);+ }+ status = cudaGetDeviceCount(&device_count);+ if (status != cudaSuccess) {+ fprintf(stderr, "cudaGetDeviceCount() failed: %s\n", cudaGetErrorString(status));+ return -1;+ }+ if (device_index >= device_count) {+ fprintf(stderr, "Specified device index %d exceeds the maximum (the device count on this system is %d)\n", device_index, device_count);+ return -1;+ }+ status = cudaGetDeviceProperties(&prop, device_index);+ if (status != cudaSuccess) {+ fprintf(stderr, "cudaGetDeviceProperties() for device device_index failed: %s\n", cudaGetErrorString(status));+ return -1;+ }+ int v = prop.major * 10 + prop.minor;+ printf("%d", v);+}diff --git a/llamafile/cuda.c b/llamafile/cuda.cnew file mode 100644index 00000000..a33e859e--- /dev/null+++ b/llamafile/cuda.c@@ -0,0 +1,532 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2023 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <cosmo.h>+#include <time.h>+#include <dlfcn.h>+#include <errno.h>+#include <spawn.h>+#include <assert.h>+#include <unistd.h>+#include <stdlib.h>+#include <limits.h>+#include <signal.h>+#include <pthread.h>+#include <sys/wait.h>+#include <sys/stat.h>+#include <stdatomic.h>+#include "llama.cpp/ggml-cuda.h"++__static_yoink("llama.cpp/ggml.h");+__static_yoink("llamafile/compcat.cu");+__static_yoink("llamafile/llamafile.h");+__static_yoink("llama.cpp/ggml-cuda.h");+__static_yoink("llama.cpp/ggml-cuda.cu");++#define NVCC_LIBS "-lcublas"++#define NVCC_FLAGS "--shared", \+ "--forward-unknown-to-host-compiler", \+ "-use_fast_math", \+ "--compiler-options", "-fPIC -O3 -march=native -mtune=native", \+ "-DNDEBUG", \+ "-DGGML_BUILD=1", \+ "-DGGML_SHARED=1", \+ "-DGGML_CUDA_DMMV_X=32", \+ "-DGGML_CUDA_MMV_Y=1", \+ "-DK_QUANTS_PER_ITERATION=2", \+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \+ "-DGGML_USE_CUBLAS"++static const struct Source {+ const char *zip;+ const char *name;+} srcs[] = {+ {"/zip/llama.cpp/ggml.h", "ggml.h"},+ {"/zip/llamafile/compcat.cu", "compcat.cu"},+ {"/zip/llamafile/llamafile.h", "llamafile.h"},+ {"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},+ {"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last+};++static struct Cuda {+ bool disabled;+ bool supported;+ atomic_uint once;+ typeof(ggml_init_cublas) *init;+ typeof(ggml_cublas_loaded) *loaded;+ typeof(ggml_cuda_host_free) *host_free;+ typeof(ggml_cuda_host_malloc) *host_malloc;+ typeof(ggml_cuda_can_mul_mat) *can_mul_mat;+ typeof(ggml_cuda_set_tensor_split) *set_tensor_split;+ typeof(ggml_cuda_transform_tensor) *transform_tensor;+ typeof(ggml_cuda_free_data) *free_data;+ typeof(ggml_cuda_assign_buffers) *assign_buffers;+ typeof(ggml_cuda_assign_buffers_no_scratch) *assign_buffers_no_scratch;+ typeof(ggml_cuda_assign_buffers_force_inplace) *assign_buffers_force_inplace;+ typeof(ggml_cuda_assign_buffers_no_alloc) *assign_buffers_no_alloc;+ typeof(ggml_cuda_assign_scratch_offset) *assign_scratch_offset;+ typeof(ggml_cuda_copy_to_device) *copy_to_device;+ typeof(ggml_cuda_set_main_device) *set_main_device;+ typeof(ggml_cuda_set_scratch_size) *set_scratch_size;+ typeof(ggml_cuda_free_scratch) *free_scratch;+ typeof(ggml_cuda_compute_forward) *compute_forward;+ typeof(ggml_cuda_get_device_count) *get_device_count;+ typeof(ggml_cuda_get_device_description) *get_device_description;+} ggml_cuda;++static const char *Dlerror(void) {+ const char *msg;+ msg = cosmo_dlerror();+ if (!msg) msg = "null dlopen error";+ return msg;+}++static const char *GetDsoExtension(void) {+ if (IsWindows()) {+ return "dll";+ } else if (IsXnu()) {+ return "dylib";+ } else {+ return "so";+ }+}++static bool FileExists(const char *path) {+ struct stat st;+ return !stat(path, &st);+}++static bool IsExecutable(const char *path) {+ struct stat st;+ return !stat(path, &st) &&+ (st.st_mode & 0111) &&+ !S_ISDIR(st.st_mode);+}++static bool CreateTempPath(const char *path, char tmp[static PATH_MAX]) {+ int fd;+ strlcpy(tmp, path, PATH_MAX);+ strlcat(tmp, ".XXXXXX", PATH_MAX);+ if ((fd = mkostemp(tmp, O_CLOEXEC)) != -1) {+ close(fd);+ return true;+ } else {+ perror(tmp);+ return false;+ }+}++static bool Compile(const char *src,+ const char *tmp,+ const char *out,+ char *args[]) {+ int pid, ws;+ errno_t err = posix_spawnp(&pid, args[0], NULL, NULL, args, environ);+ if (err) {+ perror(args[0]);+ unlink(tmp);+ return false;+ }+ while (waitpid(pid, &ws, 0) == -1) {+ if (errno != EINTR) {+ perror(args[0]);+ unlink(tmp);+ return false;+ }+ }+ if (ws) {+ tinyprint(2, args[0], ": returned nonzero exit status\n", NULL);+ unlink(tmp);+ return false;+ }+ if (rename(tmp, out)) {+ perror(out);+ unlink(tmp);+ return false;+ }+ return true;+}++// finds nvidia compiler+//+// 1. $CUDA_PATH/bin/nvcc+// 2. nvcc on $PATH environ+// 3. /opt/cuda/bin/nvcc+// 4. /usr/local/cuda/bin/nvcc+//+// set $CUDA_PATH to empty string to disable cuda+static bool GetNvccPath(char path[static PATH_MAX]) {+ const char *cuda_path;+ if ((cuda_path = getenv("CUDA_PATH"))) {+ if (!*cuda_path) return false;+ strlcpy(path, cuda_path, PATH_MAX);+ strlcat(path, "/bin/", PATH_MAX);+ } else if (commandv(IsWindows() ? "nvcc.exe" : "nvcc", path, PATH_MAX)) {+ return true;+ } else if (FileExists("/opt/cuda")) {+ strlcpy(path, "/opt/cuda/bin/", PATH_MAX);+ } else {+ strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);+ }+ strlcat(path, "nvcc", PATH_MAX);+ if (IsWindows()) {+ strlcat(path, ".exe", PATH_MAX);+ }+ return IsExecutable(path);+}++static dontinline bool GetNvccArchFlag(char *nvcc, char flag[static 32]) {++ // create path of exe+ char exe[PATH_MAX];+ llamafile_get_app_dir(exe, sizeof(exe));+ strlcat(exe, "compcat", PATH_MAX);++ // get path of sauce+ char src[PATH_MAX];+ strlcpy(src, exe, PATH_MAX);+ strlcat(src, ".cu", PATH_MAX);++ // create temporary path for output+ char tmp[PATH_MAX];+ if (!CreateTempPath(exe, tmp)) {+ return false;+ }++ // build nvidia compute capability detector+ // https://stackoverflow.com/a/40695640/1653720+ //+ // Ideally we would just say `nvcc -arch=native`, but that flag is+ // not available on devices like Jetson. If we omit the flag, then+ // llm inference prints gibberish. In order to run nvcc we need to+ // detect the microarchitecture version of the host gpu.+ //+ // nvidia-smi in cuda 11.5+ can do this but (1) it takes longer to+ // run than compiling / running this script and (2) the nvidia-smi+ // command isn't available on Jetson devices.+ tinyprint(2, "building nvidia compute capability detector...\n", NULL);+ if (!Compile(src, tmp, exe, (char *[]){nvcc, "-o", tmp, src, 0})) {+ return false;+ }++ // create pipe+ int pipefds[2];+ if (pipe2(pipefds, O_CLOEXEC)) {+ perror("pipe2");+ return false;+ }++ // run nvidia compute capability detector+ int pid;+ char *args[] = {exe, 0};+ posix_spawn_file_actions_t fa;+ posix_spawn_file_actions_init(&fa);+ posix_spawn_file_actions_adddup2(&fa, pipefds[1], 1);+ errno_t err = posix_spawn(&pid, args[0], &fa, NULL, args, environ);+ posix_spawn_file_actions_destroy(&fa);+ close(pipefds[1]);+ if (err) {+ errno = err;+ perror(args[0]);+ close(pipefds[0]);+ return false;+ }+ char ibuf[12] = {0};+ read(pipefds[0], ibuf, 11);+ close(pipefds[0]);+ int ws;+ while (waitpid(pid, &ws, 0) == -1) {+ if (errno != EINTR) {+ perror(args[0]);+ return false;+ }+ }+ if (ws) {+ tinyprint(2, "error: compute capability detector returned nonzero exit status\n", NULL);+ return false;+ }++ // parse output of detector+ char *endptr;+ if (!*ibuf || !strtol(ibuf, &endptr, 10) || *endptr) {+ tinyprint(2, "error: bad compute capability detector output\n", NULL);+ return false;+ }++ // return resulting flag+ stpcpy(stpcpy(flag, "-arch=compute_"), ibuf);+ return true;+}++static bool CompileNativeCuda(char dso[static PATH_MAX]) {++ // find full path of nvidia compiler+ char nvcc[PATH_MAX];+ if (!GetNvccPath(nvcc)) {+ tinyprint(2, "warning: couldn't find nvcc (nvidia c compiler) "+ "try setting $CUDA_PATH if it's installed\n", NULL);+ return false;+ }++ // extract source code+ char src[PATH_MAX];+ bool needs_rebuild = false;+ for (int i = 0; i < sizeof(srcs) / sizeof(*srcs); ++i) {+ llamafile_get_app_dir(src, sizeof(src));+ if (!i && mkdir(src, 0755) && errno != EEXIST) {+ perror(src);+ return false;+ }+ strlcat(src, srcs[i].name, sizeof(src));+ switch (llamafile_is_file_newer_than(srcs[i].zip, src)) {+ case -1:+ return false;+ case false:+ break;+ case true:+ needs_rebuild = true;+ if (!llamafile_extract(srcs[i].zip, src)) {+ return false;+ }+ break;+ default:+ __builtin_unreachable();+ }+ }++ // check if dso is already compiled+ llamafile_get_app_dir(dso, PATH_MAX);+ strlcat(dso, "ggml-cuda.", PATH_MAX);+ strlcat(dso, GetDsoExtension(), PATH_MAX);+ if (!needs_rebuild) {+ switch (llamafile_is_file_newer_than(src, dso)) {+ case -1:+ return false;+ case false:+ return true;+ case true:+ break;+ default:+ __builtin_unreachable();+ }+ }++ // create temporary output path for atomicity+ char tmpdso[PATH_MAX];+ if (!CreateTempPath(dso, tmpdso)) {+ return false;+ }++ // try building dso with host nvidia microarchitecture+ tinyprint(2, "building ggml-cuda with nvcc -arch=native...\n", NULL);+ if (Compile(src, tmpdso, dso, (char *[]){+ nvcc, "-arch=native", NVCC_FLAGS, "-o", tmpdso,+ src, NVCC_LIBS, NULL})) {+ return true;+ }++ // try again with different arch flag+ char archflag[32];+ if (!GetNvccArchFlag(nvcc, archflag)) {+ return false;+ }+ tinyprint(2, "building ggml-cuda with nvcc ", archflag, "...\n", NULL);+ if (Compile(src, tmpdso, dso, (char *[]){+ nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,+ src, NVCC_LIBS, NULL})) {+ return true;+ }++ // oh no+ return false;+}++static bool ImportCudaImpl(void) {++ // get native cuda dll if possible+ char dso[PATH_MAX];+ if (!CompileNativeCuda(dso)) {+ return false;+ }++ // runtime link dynamic shared object+ void *lib;+ tinyprint(2, "linking ", dso, "...\n", NULL);+ lib = cosmo_dlopen(dso, RTLD_LAZY);+ if (!lib) {+ tinyprint(2, Dlerror(), ": failed to load library\n", NULL);+ return false;+ }++ // import functions+ bool ok = true;+ ok &= !!(ggml_cuda.init = cosmo_dlsym(lib, "ggml_init_cublas"));+ ok &= !!(ggml_cuda.loaded = cosmo_dlsym(lib, "ggml_cublas_loaded"));+ ok &= !!(ggml_cuda.host_free = cosmo_dlsym(lib, "ggml_cuda_host_free"));+ ok &= !!(ggml_cuda.host_malloc = cosmo_dlsym(lib, "ggml_cuda_host_malloc"));+ ok &= !!(ggml_cuda.can_mul_mat = cosmo_dlsym(lib, "ggml_cuda_can_mul_mat"));+ ok &= !!(ggml_cuda.set_tensor_split = cosmo_dlsym(lib, "ggml_cuda_set_tensor_split"));+ ok &= !!(ggml_cuda.transform_tensor = cosmo_dlsym(lib, "ggml_cuda_transform_tensor"));+ ok &= !!(ggml_cuda.free_data = cosmo_dlsym(lib, "ggml_cuda_free_data"));+ ok &= !!(ggml_cuda.assign_buffers = cosmo_dlsym(lib, "ggml_cuda_assign_buffers"));+ ok &= !!(ggml_cuda.assign_buffers_no_scratch = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_no_scratch"));+ ok &= !!(ggml_cuda.assign_buffers_force_inplace = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_force_inplace"));+ ok &= !!(ggml_cuda.assign_buffers_no_alloc = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_no_alloc"));+ ok &= !!(ggml_cuda.assign_scratch_offset = cosmo_dlsym(lib, "ggml_cuda_assign_scratch_offset"));+ ok &= !!(ggml_cuda.copy_to_device = cosmo_dlsym(lib, "ggml_cuda_copy_to_device"));+ ok &= !!(ggml_cuda.set_main_device = cosmo_dlsym(lib, "ggml_cuda_set_main_device"));+ ok &= !!(ggml_cuda.set_scratch_size = cosmo_dlsym(lib, "ggml_cuda_set_scratch_size"));+ ok &= !!(ggml_cuda.free_scratch = cosmo_dlsym(lib, "ggml_cuda_free_scratch"));+ ok &= !!(ggml_cuda.compute_forward = cosmo_dlsym(lib, "ggml_cuda_compute_forward"));+ ok &= !!(ggml_cuda.get_device_count = cosmo_dlsym(lib, "ggml_cuda_get_device_count"));+ ok &= !!(ggml_cuda.get_device_description = cosmo_dlsym(lib, "ggml_cuda_get_device_description"));+ if (!ok) {+ tinyprint(2, Dlerror(), ": not all symbols could be imported\n", NULL);+ return false;+ }++ // we're good+ return true;+}++static void ImportCuda(void) {+ if (ggml_cuda.disabled) {+ return;+ }+ if (ImportCudaImpl()) {+ ggml_cuda.supported = true;+ tinyprint(2, "NVIDIA cuBLAS GPU supported locked and loaded\n", NULL);+ } else {+ tinyprint(2, "warning: couldn't load Nvidia CUDA GPU support\n", NULL);+ }+}++void ggml_cuda_disable(void) {+ ggml_cuda.disabled = true;+}++bool ggml_cuda_supported(void) {+ cosmo_once(&ggml_cuda.once, ImportCuda);+ return ggml_cuda.supported;+}++void ggml_init_cublas(void) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.init();+}++bool ggml_cublas_loaded(void) {+ if (!ggml_cuda_supported()) return false;+ return ggml_cuda.loaded();+}++void *ggml_cuda_host_malloc(size_t n) {+ if (!ggml_cuda_supported()) return NULL;+ return ggml_cuda.host_malloc(n);+}++void ggml_cuda_host_free(void *data) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.host_free(data);+}++bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0,+ const struct ggml_tensor *src1,+ struct ggml_tensor *dst) {+ if (!ggml_cuda_supported()) return false;+ return ggml_cuda.can_mul_mat(src0, src1, dst);+}++void ggml_cuda_set_tensor_split(const float *tensor_split) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.set_tensor_split(tensor_split);+}++void ggml_cuda_transform_tensor(void *data, struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.transform_tensor(data, tensor);+}++void ggml_cuda_free_data(struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.free_data(tensor);+}++void ggml_cuda_assign_buffers(struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.assign_buffers(tensor);+}++void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.assign_buffers_no_scratch(tensor);+}++void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.assign_buffers_force_inplace(tensor);+}++void ggml_cuda_assign_buffers_no_alloc(struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.assign_buffers_no_alloc(tensor);+}++void ggml_cuda_assign_scratch_offset(struct ggml_tensor *tensor, size_t offset) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.assign_scratch_offset(tensor, offset);+}++void ggml_cuda_copy_to_device(struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.copy_to_device(tensor);+}++void ggml_cuda_set_main_device(int main_device) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.set_main_device(main_device);+}++void ggml_cuda_set_scratch_size(size_t scratch_size) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.set_scratch_size(scratch_size);+}++void ggml_cuda_free_scratch(void) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.free_scratch();+}++bool ggml_cuda_compute_forward(struct ggml_compute_params *params,+ struct ggml_tensor *tensor) {+ if (!ggml_cuda_supported()) return false;+ return ggml_cuda.compute_forward(params, tensor);+}++int ggml_cuda_get_device_count(void) {+ if (!ggml_cuda_supported()) return 0;+ return ggml_cuda.get_device_count();+}++void ggml_cuda_get_device_description(int device,+ char *description,+ size_t description_size) {+ if (!ggml_cuda_supported()) return;+ return ggml_cuda.get_device_description(device, description,+ description_size);+}diff --git a/llamafile/extract.c b/llamafile/extract.cnew file mode 100644index 00000000..492a5b93--- /dev/null+++ b/llamafile/extract.c@@ -0,0 +1,79 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2023 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <cosmo.h>+#include <fcntl.h>+#include <stdio.h>+#include <errno.h>+#include <stdlib.h>+#include <string.h>+#include <unistd.h>+#include <sys/stat.h>+#include "llamafile.h"++/**+ * Returns true if `zip` was successfully copied to `to`.+ *+ * Copying happens atomically. The `zip` argument is a file system path,+ * which may reside under `/zip/...` to relocate a compressed executable+ * asset to the local filesystem.+ */+bool llamafile_extract(const char *zip, const char *to) {+ int fdin, fdout;+ char stage[PATH_MAX];+ tinyprint(2, "extracting ", zip, " to ", to, "\n", NULL);+ strlcpy(stage, to, sizeof(stage));+ if (strlcat(stage, ".XXXXXX", sizeof(stage)) >= sizeof(stage)) {+ errno = ENAMETOOLONG;+ perror(to);+ return false;+ }+ if ((fdout = mkostemp(stage, O_CLOEXEC)) == -1) {+ perror(stage);+ return false;+ }+ if ((fdin = open(zip, O_RDONLY | O_CLOEXEC)) == -1) {+ perror(zip);+ close(fdout);+ unlink(stage);+ return false;+ }+ if (copyfd(fdin, fdout, -1) == -1) {+ perror(zip);+ close(fdin);+ close(fdout);+ unlink(stage);+ return false;+ }+ if (close(fdout)) {+ perror(to);+ close(fdin);+ unlink(stage);+ return false;+ }+ if (close(fdin)) {+ perror(zip);+ unlink(stage);+ return false;+ }+ if (rename(stage, to)) {+ perror(to);+ unlink(stage);+ return false;+ }+ return true;+}diff --git a/llamafile/get_app_dir.c b/llamafile/get_app_dir.cnew file mode 100644index 00000000..534edf7d--- /dev/null+++ b/llamafile/get_app_dir.c@@ -0,0 +1,27 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2023 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llamafile.h"+#include <string.h>++/**+ * Returns path of directory for app-specific files.+ */+void llamafile_get_app_dir(char *path, size_t size) {+ strlcpy(path, llamafile_get_tmp_dir(), size);+ strlcat(path, "/.llamafile/", size);+}diff --git a/llama.cpp/llava/llava-quantize.cc b/llamafile/get_tmp_dir.csimilarity index 59%rename from llama.cpp/llava/llava-quantize.ccrename to llamafile/get_tmp_dir.cindex 93be174d..f0b4e855 100644--- a/llama.cpp/llava/llava-quantize.cc+++ b/llamafile/get_tmp_dir.c@@ -1,5 +1,5 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set net ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi//// Copyright 2023 Mozilla Foundation//@@ -15,18 +15,15 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <stdio.h>+#include "llamafile.h"#include <stdlib.h>-#include "llama.cpp/ggml.h"-#include "clip.h"-int main(int argc, char *argv[]) {- ggml_check_cpu();- if (argc != 4) {- fprintf(stderr, "Usage: %s INPUT OUTPUT FORMAT\n");- exit(1);- }- if (!clip_model_quantize(argv[1], argv[2], atoi(argv[3]))) {- exit(1);+const char *llamafile_get_tmp_dir(void) {+ const char *tmpdir;+ if (!(tmpdir = getenv("TMPDIR")) || !*tmpdir) {+ if (!(tmpdir = getenv("HOME")) || !*tmpdir) {+ tmpdir = ".";+ }}+ return tmpdir;}diff --git a/llama.cpp/extract.c b/llamafile/is_file_newer_than.csimilarity index 61%rename from llama.cpp/extract.crename to llamafile/is_file_newer_than.cindex bd0b88d0..d9f6315d 100644--- a/llama.cpp/extract.c+++ b/llamafile/is_file_newer_than.c@@ -15,8 +15,7 @@// See the License for the specific language governing permissions and// limitations under the License.-#define _COSMO_SOURCE-#include <cosmo.h>+#include <time.h>#include <fcntl.h>#include <stdio.h>#include <errno.h>@@ -24,27 +23,9 @@#include <string.h>#include <unistd.h>#include <sys/stat.h>-#include "ggml.h"+#include "llamafile.h"-static const char *ggml_get_tmp_dir(void) {- const char *tmpdir;- if (!(tmpdir = getenv("TMPDIR")) || !*tmpdir) {- if (!(tmpdir = getenv("HOME")) || !*tmpdir) {- tmpdir = ".";- }- }- return tmpdir;-}--/**- * Returns path of directory for app-specific files.- */-void ggml_get_app_dir(char *path, size_t size) {- strlcpy(path, ggml_get_tmp_dir(), size);- strlcat(path, "/.llamafile/", size);-}--static int ggml_is_file_newer_than_time(const char *path, const char *other) {+static int is_file_newer_than_time(const char *path, const char *other) {struct stat st1, st2;if (stat(path, &st1)) {// PATH should always exist when calling this function@@ -65,7 +46,7 @@ static int ggml_is_file_newer_than_time(const char *path, const char *other) {return timespec_cmp(st1.st_mtim, st2.st_mtim) > 0;}-static int ggml_is_file_newer_than_bytes(const char *path, const char *other) {+static int is_file_newer_than_bytes(const char *path, const char *other) {int other_fd;if ((other_fd = open(other, O_RDONLY | O_CLOEXEC)) == -1) {if (errno == ENOENT) {@@ -135,65 +116,12 @@ static int ggml_is_file_newer_than_bytes(const char *path, const char *other) {* source code file (which may reside under `/zip/...`, and `other`* would be the generated artifact that's dependent on `path`.*/-int ggml_is_file_newer_than(const char *path, const char *other) {+int llamafile_is_file_newer_than(const char *path, const char *other) {if (startswith(path, "/zip/")) {// to keep builds deterministic, embedded zip files always have// the same timestamp from back in 2022 when it was implemented- return ggml_is_file_newer_than_bytes(path, other);+ return is_file_newer_than_bytes(path, other);} else {- return ggml_is_file_newer_than_time(path, other);- }-}--/**- * Returns true if `zip` was successfully copied to `to`.- *- * Copying happens atomically. The `zip` argument is a file system path,- * which may reside under `/zip/...` to relocate a compressed executable- * asset to the local filesystem.- */-bool ggml_extract(const char *zip, const char *to) {- int fdin, fdout;- char stage[PATH_MAX];- tinyprint(2, "extracting ", zip, " to ", to, "\n", NULL);- strlcpy(stage, to, sizeof(stage));- if (strlcat(stage, ".XXXXXX", sizeof(stage)) >= sizeof(stage)) {- errno = ENAMETOOLONG;- perror(to);- return false;- }- if ((fdout = mkostemp(stage, O_CLOEXEC)) == -1) {- perror(stage);- return false;- }- if ((fdin = open(zip, O_RDONLY | O_CLOEXEC)) == -1) {- perror(zip);- close(fdout);- unlink(stage);- return false;- }- if (copyfd(fdin, fdout, -1) == -1) {- perror(zip);- close(fdin);- close(fdout);- unlink(stage);- return false;- }- if (close(fdout)) {- perror(to);- close(fdin);- unlink(stage);- return false;- }- if (close(fdin)) {- perror(zip);- unlink(stage);- return false;- }- if (rename(stage, to)) {- perror(to);- unlink(stage);- return false;+ return is_file_newer_than_time(path, other);}- return true;}diff --git a/llama.cpp/ggml-file.c b/llamafile/llamafile.csimilarity index 87%rename from llama.cpp/ggml-file.crename to llamafile/llamafile.cindex af8f798d..7b617821 100644--- a/llama.cpp/ggml-file.c+++ b/llamafile/llamafile.c@@ -15,28 +15,28 @@// See the License for the specific language governing permissions and// limitations under the License.-#define _COSMO_SOURCE#include <cosmo.h>#include <errno.h>#include <fcntl.h>#include <stdio.h>+#include <assert.h>#include <unistd.h>#include <stdlib.h>#include <stdint.h>#include <limits.h>#include <unistd.h>#include <sys/mman.h>-#include "ggml.h"+#include "llamafile.h"#include "zip.h"#define Min(a, b) ((a) < (b) ? (a) : (b))-struct ggml_file {- FILE * fp;+struct llamafile {+ FILE *fp;size_t size;- char * content;+ char *content;size_t position;- void * mapping;+ void *mapping;size_t mapsize;};@@ -70,22 +70,22 @@ static uint64_t GetZipCfileCompressedSize(const uint8_t *z) {return x;}-struct ggml_file *ggml_file_open(const char * fname, const char * mode) {+struct llamafile *llamafile_open(const char *fname, const char *mode) {int fd = -1;- uint8_t * bufdata = NULL;+ uint8_t *bufdata = NULL;size_t cdirsize = 0;- uint8_t * cdirdata = NULL;- struct ggml_file * file = NULL;+ uint8_t *cdirdata = NULL;+ struct llamafile *file = NULL;- if (!(file = malloc(sizeof(struct ggml_file)))) {+ if (!(file = malloc(sizeof(struct llamafile)))) {goto Failure;}// open from the filesystem if it existsif ((file->fp = fopen(fname, mode))) {- ggml_file_seek(file, 0, SEEK_END);- file->size = ggml_file_tell(file);- ggml_file_seek(file, 0, SEEK_SET);+ llamafile_seek(file, 0, SEEK_END);+ file->size = llamafile_tell(file);+ llamafile_seek(file, 0, SEEK_SET);return file;}if (errno != ENOENT) {@@ -222,10 +222,6 @@ struct ggml_file *ggml_file_open(const char * fname, const char * mode) {fname, prog, strerror(errno));goto Failure;}- if (ggml_is_numa() && posix_madvise(file->mapping, file->mapsize, POSIX_MADV_RANDOM)) {- fprintf(stderr, "warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",- strerror(errno));- }// setup our synthetic filefile->position = 0;@@ -247,19 +243,19 @@ Failure:return 0;}-FILE * ggml_file_fp(struct ggml_file * file) {+FILE *llamafile_fp(struct llamafile *file) {return file->fp;}-size_t ggml_file_size(struct ggml_file * file) {+size_t llamafile_size(struct llamafile *file) {return file->size;}-void * ggml_file_content(struct ggml_file * file) {+void *llamafile_content(struct llamafile *file) {return file->content;}-size_t ggml_file_tell(struct ggml_file * file) {+size_t llamafile_tell(struct llamafile *file) {if (!file->fp) {return file->position;}@@ -268,11 +264,11 @@ size_t ggml_file_tell(struct ggml_file * file) {#elselong ret = ftell(file->fp);#endif- GGML_ASSERT(ret != -1); // this really shouldn't fail+ unassert(ret != -1); // this really shouldn't failreturn (size_t) ret;}-void ggml_file_seek(struct ggml_file * file, size_t offset, int whence) {+void llamafile_seek(struct llamafile *file, size_t offset, int whence) {if (!file->fp) {switch (whence) {case SEEK_SET:@@ -287,15 +283,10 @@ void ggml_file_seek(struct ggml_file * file, size_t offset, int whence) {}return;}-#ifdef _WIN32- int ret = _fseeki64(file->fp, (__int64) offset, whence);-#else- int ret = fseek(file->fp, (long) offset, whence);-#endif- GGML_ASSERT(ret == 0); // same+ unassert(!fseek(file->fp, (long) offset, whence));}-long ggml_file_read(struct ggml_file * file, void * ptr, size_t len) {+long llamafile_read(struct llamafile *file, void *ptr, size_t len) {if (len == 0) {return 0;}@@ -320,7 +311,7 @@ long ggml_file_read(struct ggml_file * file, void * ptr, size_t len) {return len;}-long ggml_file_write(struct ggml_file * file, const void * ptr, size_t len) {+long llamafile_write(struct llamafile *file, const void *ptr, size_t len) {if (len == 0) {return 0;}@@ -339,7 +330,7 @@ long ggml_file_write(struct ggml_file * file, const void * ptr, size_t len) {return len;}-void ggml_file_close(struct ggml_file * file) {+void llamafile_close(struct llamafile *file) {if (file->fp) {fclose(file->fp);}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hnew file mode 100644index 00000000..2809ffa4--- /dev/null+++ b/llamafile/llamafile.h@@ -0,0 +1,30 @@+#ifndef LLAMAFILE_H_+#define LLAMAFILE_H_+#include <stdio.h>+#include <stdbool.h>+#ifdef __cplusplus+extern "C" {+#endif++struct llamafile;+struct llamafile *llamafile_open(const char *, const char *);+void llamafile_close(struct llamafile *);+long llamafile_read(struct llamafile *, void *, size_t);+long llamafile_write(struct llamafile *, const void *, size_t);+void llamafile_seek(struct llamafile *, size_t, int);+void *llamafile_content(struct llamafile *);+size_t llamafile_tell(struct llamafile *);+size_t llamafile_size(struct llamafile *);+FILE *llamafile_fp(struct llamafile *);++void llamafile_check_cpu(void);+const char *llamafile_get_tmp_dir(void);+bool llamafile_extract(const char *, const char *);+int llamafile_is_file_newer_than(const char *, const char *);+void llamafile_schlep(const void *, size_t);+void llamafile_get_app_dir(char *, size_t);++#ifdef __cplusplus+}+#endif+#endif /* LLAMAFILE_H_ */diff --git a/llama.cpp/metal.c b/llamafile/metal.csimilarity index 95%rename from llama.cpp/metal.crename to llamafile/metal.cindex 57f0022a..0a5af6fa 100644--- a/llama.cpp/metal.c+++ b/llamafile/metal.c@@ -15,7 +15,6 @@// See the License for the specific language governing permissions and// limitations under the License.-#define _COSMO_SOURCE#include <cosmo.h>#include <time.h>#include <dlfcn.h>@@ -29,9 +28,10 @@#include <sys/wait.h>#include <sys/stat.h>#include <stdatomic.h>-#include "ggml-metal.h"+#include "llama.cpp/ggml-metal.h"__static_yoink("llama.cpp/ggml.h");+__static_yoink("llamafile/llamafile.h");__static_yoink("llama.cpp/ggml-impl.h");__static_yoink("llama.cpp/ggml-metal.m");__static_yoink("llama.cpp/ggml-metal.h");@@ -43,6 +43,7 @@ static const struct Source {const char *name;} srcs[] = {{"/zip/llama.cpp/ggml.h", "ggml.h"},+ {"/zip/llamafile/llamafile.h", "llamafile.h"},{"/zip/llama.cpp/ggml-impl.h", "ggml-impl.h"},{"/zip/llama.cpp/ggml-metal.h", "ggml-metal.h"},{"/zip/llama.cpp/ggml-quants.h", "ggml-quants.h"},@@ -78,20 +79,20 @@ static bool ImportMetalImpl(void) {char src[PATH_MAX];bool needs_rebuild = false;for (int i = 0; i < sizeof(srcs) / sizeof(*srcs); ++i) {- ggml_get_app_dir(src, PATH_MAX);+ llamafile_get_app_dir(src, PATH_MAX);if (!i && mkdir(src, 0755) && errno != EEXIST) {perror(src);return false;}strlcat(src, srcs[i].name, sizeof(src));- switch (ggml_is_file_newer_than(srcs[i].zip, src)) {+ switch (llamafile_is_file_newer_than(srcs[i].zip, src)) {case -1:return false;case 0:break;case 1:needs_rebuild = true;- if (!ggml_extract(srcs[i].zip, src)) {+ if (!llamafile_extract(srcs[i].zip, src)) {return false;}break;@@ -102,10 +103,10 @@ static bool ImportMetalImpl(void) {// determine if we need to buildchar dso[PATH_MAX];- ggml_get_app_dir(dso, PATH_MAX);+ llamafile_get_app_dir(dso, PATH_MAX);strlcat(dso, "ggml-metal.dylib", sizeof(dso));if (!needs_rebuild) {- switch (ggml_is_file_newer_than(src, dso)) {+ switch (llamafile_is_file_newer_than(src, dso)) {case -1:return false;case 0:@@ -230,7 +231,7 @@ struct ggml_metal_context *ggml_metal_init(int n_cb, const char *metalPath) {char path[PATH_MAX];if (!ggml_metal_supported()) return NULL;if (!metalPath) {- ggml_get_app_dir(path, PATH_MAX);+ llamafile_get_app_dir(path, PATH_MAX);strlcat(path, "ggml-metal.metal", sizeof(path));metalPath = path;}diff --git a/llama.cpp/schlep.c b/llamafile/schlep.csimilarity index 97%rename from llama.cpp/schlep.crename to llamafile/schlep.cindex 3004b0a5..ccd17f34 100644--- a/llama.cpp/schlep.c+++ b/llamafile/schlep.c@@ -15,7 +15,6 @@// See the License for the specific language governing permissions and// limitations under the License.-#define _COSMO_SOURCE#include <cosmo.h>#include <stdio.h>#include <errno.h>@@ -23,7 +22,7 @@#include <pthread.h>#include <sys/auxv.h>#include <stdatomic.h>-#include "ggml.h"+#include "llama.cpp/ggml.h"#define FPS 24@@ -67,7 +66,7 @@ static void *ProgressReporter(void *arg) {/*** Loads memory off disk while reporting progress.*/-void ggml_schlep(const void *data, size_t size) {+void llamafile_schlep(const void *data, size_t size) {// don't bother if memory is smallif (size < 128 * 1024 * 1024) {diff --git a/llama.cpp/zip.h b/llamafile/zip.hsimilarity index 100%rename from llama.cpp/zip.hrename to llamafile/zip.h