Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Clean up more small issues
Justine Tunney7be970af30b7
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 56c81340..8007d012 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -815,14 +815,21 @@ struct llama_mmap {// therefore it's already mappedis_owned = false;addr = llamafile_content(file->file);- llamafile_schlep(addr, size);+ if (!llamafile_gpu_supported()) {+ llamafile_schlep(addr, size);+ }return;}is_owned = true;int fd = fileno(llamafile_fp(file->file));+ // advise the kernel to read the file sequentially (increases readahead)+ if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {+ LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",+ strerror(errno));+ }// prefetch/readahead impairs performance on NUMA systemsif (numa) { prefetch = 0; }- addr = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);+ addr = mmap(NULL, size, PROT_READ, MAP_SHARED | MAP_POPULATE, fd, 0);if (addr == MAP_FAILED) {ThrowRuntimeError(format("mmap failed: %s", strerror(errno)));}@@ -886,9 +893,12 @@ struct llama_mmap {void * next_page_start = (uint8_t *) addr + first;// unmap the range+#if 0+ // TODO(jart): make this safeif (munmap(next_page_start, len)) {LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));}+#endif// update the list of mapped fragments to avoid unmapping the same range again in the destructorstd::vector<std::pair<size_t, size_t>> new_mapped_fragments;diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 29049fd2..56d1400a 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -98,14 +98,14 @@ static struct Cuda {bool supported;atomic_uint once;typeof(ggml_cuda_link) *ggml_cuda_link;- typeof(ggml_init_cublas) *init;- typeof(ggml_cublas_loaded) *loaded;- typeof(ggml_cuda_host_free) *host_free;- typeof(ggml_cuda_host_malloc) *host_malloc;+ typeof(ggml_init_cublas) *ggml_init_cublas;+ typeof(ggml_cublas_loaded) *ggml_cublas_loaded;+ typeof(ggml_cuda_host_free) *ggml_cuda_host_free;+ typeof(ggml_cuda_host_malloc) *ggml_cuda_host_malloc;typeof(ggml_cuda_can_mul_mat) *can_mul_mat;typeof(ggml_cuda_set_tensor_split) *set_tensor_split;- typeof(ggml_cuda_transform_tensor) *transform_tensor;- typeof(ggml_cuda_free_data) *free_data;+ typeof(ggml_cuda_transform_tensor) *ggml_cuda_transform_tensor;+ typeof(ggml_cuda_free_data) *ggml_cuda_free_data;typeof(ggml_cuda_assign_buffers) *assign_buffers;typeof(ggml_cuda_assign_buffers_no_scratch) *assign_buffers_no_scratch;typeof(ggml_cuda_assign_buffers_force_inplace) *assign_buffers_force_inplace;@@ -118,10 +118,10 @@ static struct Cuda {typeof(ggml_cuda_compute_forward) *compute_forward;typeof(ggml_cuda_get_device_count) *get_device_count;typeof(ggml_cuda_get_device_description) *get_device_description;- typeof(ggml_backend_cuda_buffer_type) *buffer_type;+ typeof(ggml_backend_cuda_buffer_type) *ggml_backend_cuda_buffer_type;typeof(ggml_backend_reg_cuda_init) *backend_reg_init;- typeof(ggml_backend_cuda_host_buffer_type) *backend_host_buffer_type;- typeof(ggml_backend_cuda_init) *backend_init;+ typeof(ggml_backend_cuda_host_buffer_type) *ggml_backend_cuda_host_buffer_type;+ typeof(ggml_backend_cuda_init) *ggml_backend_cuda_init;} ggml_cuda;static const char *Dlerror(void) {@@ -606,14 +606,14 @@ static bool LinkCudaDso(const char *dso, const char *dir) {// import functionsbool ok = true;ok &= !!(ggml_cuda.ggml_cuda_link = cosmo_dlsym(lib, "ggml_cuda_link"));- ok &= !!(ggml_cuda.init = cosmo_dlsym(lib, "ggml_init_cublas"));- ok &= !!(ggml_cuda.loaded = cosmo_dlsym(lib, "ggml_cublas_loaded"));- ok &= !!(ggml_cuda.host_free = cosmo_dlsym(lib, "ggml_cuda_host_free"));- ok &= !!(ggml_cuda.host_malloc = cosmo_dlsym(lib, "ggml_cuda_host_malloc"));+ ok &= !!(ggml_cuda.ggml_init_cublas = cosmo_dlsym(lib, "ggml_init_cublas"));+ ok &= !!(ggml_cuda.ggml_cublas_loaded = cosmo_dlsym(lib, "ggml_cublas_loaded"));+ ok &= !!(ggml_cuda.ggml_cuda_host_free = cosmo_dlsym(lib, "ggml_cuda_host_free"));+ ok &= !!(ggml_cuda.ggml_cuda_host_malloc = cosmo_dlsym(lib, "ggml_cuda_host_malloc"));ok &= !!(ggml_cuda.can_mul_mat = cosmo_dlsym(lib, "ggml_cuda_can_mul_mat"));ok &= !!(ggml_cuda.set_tensor_split = cosmo_dlsym(lib, "ggml_cuda_set_tensor_split"));- ok &= !!(ggml_cuda.transform_tensor = cosmo_dlsym(lib, "ggml_cuda_transform_tensor"));- ok &= !!(ggml_cuda.free_data = cosmo_dlsym(lib, "ggml_cuda_free_data"));+ ok &= !!(ggml_cuda.ggml_cuda_transform_tensor = cosmo_dlsym(lib, "ggml_cuda_transform_tensor"));+ ok &= !!(ggml_cuda.ggml_cuda_free_data = cosmo_dlsym(lib, "ggml_cuda_free_data"));ok &= !!(ggml_cuda.assign_buffers = cosmo_dlsym(lib, "ggml_cuda_assign_buffers"));ok &= !!(ggml_cuda.assign_buffers_no_scratch = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_no_scratch"));ok &= !!(ggml_cuda.assign_buffers_force_inplace = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_force_inplace"));@@ -627,8 +627,9 @@ static bool LinkCudaDso(const char *dso, const char *dir) {ok &= !!(ggml_cuda.get_device_count = cosmo_dlsym(lib, "ggml_cuda_get_device_count"));ok &= !!(ggml_cuda.get_device_description = cosmo_dlsym(lib, "ggml_cuda_get_device_description"));ok &= !!(ggml_cuda.backend_reg_init = cosmo_dlsym(lib, "ggml_backend_reg_cuda_init"));- ok &= !!(ggml_cuda.backend_host_buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_host_buffer_type"));- ok &= !!(ggml_cuda.backend_init = cosmo_dlsym(lib, "ggml_backend_cuda_init"));+ ok &= !!(ggml_cuda.ggml_backend_cuda_host_buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_host_buffer_type"));+ ok &= !!(ggml_cuda.ggml_backend_cuda_buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_buffer_type"));+ ok &= !!(ggml_cuda.ggml_backend_cuda_init = cosmo_dlsym(lib, "ggml_backend_cuda_init"));if (!ok) {tinylog("error: not all cuda symbols could be imported\n", NULL);return false;@@ -848,22 +849,22 @@ bool ggml_cuda_supported(void) {void ggml_init_cublas(void) {if (!ggml_cuda_supported()) return;- return ggml_cuda.init();+ return ggml_cuda.ggml_init_cublas();}bool ggml_cublas_loaded(void) {if (!ggml_cuda_supported()) return false;- return ggml_cuda.loaded();+ return ggml_cuda.ggml_cublas_loaded();}void *ggml_cuda_host_malloc(size_t n) {if (!ggml_cuda_supported()) return NULL;- return ggml_cuda.host_malloc(n);+ return ggml_cuda.ggml_cuda_host_malloc(n);}void ggml_cuda_host_free(void *data) {if (!ggml_cuda_supported()) return;- return ggml_cuda.host_free(data);+ return ggml_cuda.ggml_cuda_host_free(data);}bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0,@@ -880,12 +881,12 @@ void ggml_cuda_set_tensor_split(const float *tensor_split) {void ggml_cuda_transform_tensor(void *data, struct ggml_tensor *tensor) {if (!ggml_cuda_supported()) return;- return ggml_cuda.transform_tensor(data, tensor);+ return ggml_cuda.ggml_cuda_transform_tensor(data, tensor);}void ggml_cuda_free_data(struct ggml_tensor *tensor) {if (!ggml_cuda_supported()) return;- return ggml_cuda.free_data(tensor);+ return ggml_cuda.ggml_cuda_free_data(tensor);}void ggml_cuda_assign_buffers(struct ggml_tensor *tensor) {@@ -954,7 +955,7 @@ void ggml_cuda_get_device_description(int device,ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {if (!ggml_cuda_supported()) return 0;- return ggml_cuda.buffer_type(device);+ return ggml_cuda.ggml_backend_cuda_buffer_type(device);}ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {@@ -964,7 +965,7 @@ ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data)ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {if (!ggml_cuda_supported()) return 0;- ggml_cuda.backend_host_buffer_type();+ ggml_cuda.ggml_backend_cuda_host_buffer_type();}int ggml_backend_cuda_reg_devices(void) {@@ -980,5 +981,5 @@ int ggml_backend_cuda_reg_devices(void) {ggml_backend_t ggml_backend_cuda_init(int device) {if (!ggml_cuda_supported()) return 0;- return ggml_cuda.backend_init(device);+ return ggml_cuda.ggml_backend_cuda_init(device);}diff --git a/llamafile/gpu.c b/llamafile/gpu.cindex ca53b93a..79cc0b75 100644--- a/llamafile/gpu.c+++ b/llamafile/gpu.c@@ -50,7 +50,7 @@ const char *llamafile_describe_gpu(void) {* Returns true if GPU support is available.*/int llamafile_gpu_supported(void) {- return ggml_metal_supported() && ggml_cuda_supported();+ return ggml_metal_supported() || ggml_cuda_supported();}/**