Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make zipalign and slicehf gentler on system
Justine Tunney909f791666fb
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex ca9fb3b6..abcd9c9e 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -1567,7 +1567,8 @@ void ggml_once(atomic_uint * once, void init(void)) {}int ggml_delay(int backoff) {- if (backoff < 12) {+#if 0+ if (backoff < 7) {volatile int i;for (i = 0; i != 1 << backoff; i++) {}@@ -1575,6 +1576,7 @@ int ggml_delay(int backoff) {} else {pthread_yield_np();}+#endifreturn backoff;}@@ -1587,9 +1589,9 @@ void ggml_syncthreads(struct ggml_barrier * b, int nth) {atomic_store_explicit(&b->phase, phase + 1, memory_order_release);} else {int backoff = 0;- while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase) {- backoff = ggml_delay(backoff);- }+ while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase)+ while (atomic_load_explicit(&b->phase, memory_order_relaxed) == phase)+ backoff = ggml_delay(backoff);}sync_wait_end();}@@ -18682,7 +18684,11 @@ static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_computwhile (true) {* node_n = atomic_load_explicit(&state->shared->node_n, memory_order_acquire);if (* node_n != last_node_n) break;- backoff = ggml_delay(backoff);+ while (true) {+ * node_n = atomic_load_explicit(&state->shared->node_n, memory_order_relaxed);+ if (* node_n != last_node_n) break;+ backoff = ggml_delay(backoff);+ }}sync_wait_end();}@@ -18696,7 +18702,11 @@ static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_cowhile (true) {* task_phase = atomic_load_explicit(&state->shared->node_task, memory_order_acquire);if (* task_phase != last_task_phase) break;- backoff = ggml_delay(backoff);+ while (true) {+ * task_phase = atomic_load_explicit(&state->shared->node_task, memory_order_relaxed);+ if (* task_phase != last_task_phase) break;+ backoff = ggml_delay(backoff);+ }}sync_wait_end();}diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex bf1c62d4..aad9f494 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -3590,7 +3590,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {}}-static const char * llama_model_type_name(e_model type) {+static const char * llama_model_type_name(char *buf, size_t size, e_model type) {switch (type) {case MODEL_22M: return "22M";case MODEL_33M: return "33M";@@ -3622,7 +3622,9 @@ static const char * llama_model_type_name(e_model type) {case MODEL_8x7B: return "8x7B";case MODEL_8x22B: return "8x22B";case MODEL_16x12B: return "16x12B";- default: return "?B";+ default:+ snprintf(buf, size, "?%d", type);+ return buf;}}@@ -4554,6 +4556,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {const auto & vocab = model.vocab;const char * rope_scaling_type = LLAMA_ROPE_SCALING_TYPES.at(hparams.rope_scaling_type_train);+ char buf[64];// hparamsLLAMA_LOG_INFO("%s: format = %s\n", __func__, llama_file_version_name(ml.fver));@@ -4593,7 +4596,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);LLAMA_LOG_INFO("%s: ssm_dt_rank = %u\n", __func__, hparams.ssm_dt_rank);- LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type));+ LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(buf, sizeof(buf), model.type));LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str());if (ml.n_elements >= 1e12) {LLAMA_LOG_INFO("%s: model params = %.2f T\n", __func__, ml.n_elements*1e-12);@@ -15976,9 +15979,10 @@ int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int3}int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {+ char buf2[64];return snprintf(buf, buf_size, "%s %s %s",llama_model_arch_name(model->arch),- llama_model_type_name(model->type),+ llama_model_type_name(buf2, sizeof(buf2), model->type),llama_model_ftype_name(model->ftype).c_str());}diff --git a/llamafile/slicehf.c b/llamafile/slicehf.cindex 8ab42d2c..9463134f 100644--- a/llamafile/slicehf.c+++ b/llamafile/slicehf.c@@ -26,6 +26,8 @@#define MAX_SIZE (50L * 1000 * 1000 * 1000)+static char in_path_noext[PATH_MAX + 1];+static wontreturn void Die(const char *thing, const char *reason) {tinyprint(2, thing, ": ", reason, "\n", NULL);exit(1);@@ -37,9 +39,19 @@ static wontreturn void DieSys(const char *thing) {}void slicehf(const char *in_path) {++ off_t in_off = 0;int in_fd = open(in_path, O_RDONLY);if (in_fd == -1)DieSys(in_path);+ posix_fadvise(in_fd, 0, 0, POSIX_FADV_SEQUENTIAL);++ // split file extension+ char *ext;+ strlcpy(in_path_noext, in_path, sizeof(in_path_noext));+ if (!(ext = strrchr(in_path_noext, '.')) || strchr(ext, '/'))+ Die(in_path, "file missing extension");+ *ext++ = '\0';struct stat st;if (fstat(in_fd, &st) == -1)@@ -55,17 +67,19 @@ void slicehf(const char *in_path) {long amount = MIN(total, MAX_SIZE);total -= amount;char out_path[PATH_MAX];- snprintf(out_path, PATH_MAX, "%s.cat%d", in_path, item++);+ snprintf(out_path, PATH_MAX, "%s.cat%d.%s", in_path_noext, item++, ext);+ off_t out_off = 0;int out_fd = open(out_path, O_WRONLY | O_TRUNC | O_CREAT, 0644);if (out_fd == -1)DieSys(out_path);while (amount) {- long chunk = MIN(amount, 65536);- long copied = copy_file_range(in_fd, 0, out_fd, 0, chunk, 0);+ long chunk = MIN(amount, 2097152);+ long copied = copy_file_range(in_fd, &in_off, out_fd, &out_off, chunk, 0);if (copied == -1)DieSys(out_path);if (!copied)Die(in_path, "unexpected eof");+ posix_fadvise(in_fd, in_off - copied, copied, POSIX_FADV_DONTNEED);amount -= copied;}close(out_fd);@@ -75,7 +89,10 @@ void slicehf(const char *in_path) {}int main(int argc, char *argv[]) {- for (int i = 1; i < argc; ++i) {++ // use idle scheduling priority+ verynice();++ for (int i = 1; i < argc; ++i)slicehf(argv[i]);- }}diff --git a/llamafile/zipalign.c b/llamafile/zipalign.cindex 5fdf3304..a3775cb8 100644--- a/llamafile/zipalign.c+++ b/llamafile/zipalign.c@@ -142,6 +142,9 @@ int main(int argc, char *argv[]) {if (optind == argc)Die(prog, "missing output argument");+ // use idle scheduling priority+ verynice();+// open output fileint zfd;ssize_t zsize;@@ -257,6 +260,7 @@ int main(int argc, char *argv[]) {const char *path = argv[i];if ((fd = open(path, O_RDONLY)) == -1)DieSys(path);+ posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);// get information about fileuint64_t size;@@ -315,6 +319,7 @@ int main(int argc, char *argv[]) {// read chunkif ((rc = pread(fd, iobuf, Min(size, CHUNK), i)) <= 0)DieSys(path);+ posix_fadvise(fd, i, Min(size, CHUNK), POSIX_FADV_DONTNEED);crc = crc32(crc, iobuf, rc);if (!flag_level) {// write uncompressed chunk to output