Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make some more benchmark tool fixes
Justine Tunneye67571914779
diff --git a/README.md b/README.mdindex 54db8942..4319025a 100644--- a/README.md+++ b/README.md@@ -175,7 +175,7 @@ try out llamafile with different kinds of LLMs.| Mistral-7B-Instruct | 3.85 GB | [Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) | [mistral-7b-instruct-v0.2.Q4\_0.llamafile](https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q4_0.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile) || Phi-3-mini-4k-instruct | 7.67 GB | [Apache 2.0](https://huggingface.co/Mozilla/Phi-3-mini-4k-instruct-llamafile/blob/main/LICENSE) | [Phi-3-mini-4k-instruct.F16.llamafile](https://huggingface.co/Mozilla/Phi-3-mini-4k-instruct-llamafile/resolve/main/Phi-3-mini-4k-instruct.F16.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Phi-3-mini-4k-instruct-llamafile) || Mixtral-8x7B-Instruct | 30.03 GB | [Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) | [mixtral-8x7b-instruct-v0.1.Q5\_K\_M.llamafile](https://huggingface.co/Mozilla/Mixtral-8x7B-Instruct-v0.1-llamafile/resolve/main/mixtral-8x7b-instruct-v0.1.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Mixtral-8x7B-Instruct-v0.1-llamafile) |-| WizardCoder-Python-34B | 22.23 GB | [LLaMA 2](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) | [wizardcoder-python-34b-v1.0.Q5\_K\_M.llamafile](https://huggingface.co/jartine/WizardCoder-Python-34B-V1.0-llamafile/resolve/main/wizardcoder-python-34b-v1.0.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/jartine/WizardCoder-Python-34B-V1.0-llamafile) |+| WizardCoder-Python-34B | 22.23 GB | [LLaMA 2](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) | [wizardcoder-python-34b-v1.0.Q5\_K\_M.llamafile](https://huggingface.co/Mozilla/WizardCoder-Python-34B-V1.0-llamafile/resolve/main/wizardcoder-python-34b-v1.0.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/WizardCoder-Python-34B-V1.0-llamafile) || WizardCoder-Python-13B | 7.33 GB | [LLaMA 2](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) | [wizardcoder-python-13b.llamafile](https://huggingface.co/jartine/wizardcoder-13b-python/resolve/main/wizardcoder-python-13b.llamafile?download=true) | [See HF repo](https://huggingface.co/jartine/wizardcoder-13b-python) || LLaMA-3-Instruct-70B | 37.25 GB | [llama3](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/blob/main/Meta-Llama-3-Community-License-Agreement.txt) | [Meta-Llama-3-70B-Instruct.Q4\_0.llamafile](https://huggingface.co/Mozilla/Meta-Llama-3-70B-Instruct-llamafile/resolve/main/Meta-Llama-3-70B-Instruct.Q4_0.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Meta-Llama-3-70B-Instruct-llamafile) || LLaMA-3-Instruct-8B | 5.37 GB | [llama3](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/blob/main/Meta-Llama-3-Community-License-Agreement.txt) | [Meta-Llama-3-8B-Instruct.Q5\_K\_M.llamafile](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile) |diff --git a/llama.cpp/llama-bench/llama-bench.cpp b/llama.cpp/llama-bench/llama-bench.cppindex c335cb82..c436a431 100644--- a/llama.cpp/llama-bench/llama-bench.cpp+++ b/llama.cpp/llama-bench/llama-bench.cpp@@ -19,6 +19,9 @@#include <cosmo.h>#include <libgen.h>#include <sys/stat.h>+#include <sys/auxv.h>+#include <libc/intrin/x86.h>+#include <libc/sysv/consts/hwcap.h>#include "llama.cpp/ggml.h"#include "llama.cpp/llama.h"@@ -85,8 +88,18 @@ static T stdev(const std::vector<T> & v) {return stdev;}-static std::string get_cpu_info() {+static std::string replaceAll(std::string str, const std::string& from, const std::string& to) {+ size_t start_pos = 0;+ while ((start_pos = str.find(from, start_pos)) != std::string::npos) {+ str.replace(start_pos, from.length(), to);+ start_pos += to.length(); // Handles case where 'to' is a substring of 'from'+ }+ return str;+}++static std::string get_cpu_info() { // [jart]std::string id;+if (IsLinux()) {FILE * f = fopen("/proc/cpuinfo", "r");if (f) {@@ -104,20 +117,46 @@ static std::string get_cpu_info() {p[strlen(p) - 1] = '\0';}id = p;+ id = replaceAll(id, " 96-Cores", "");+ id = replaceAll(id, "(TM)", "");+ id = replaceAll(id, "(R)", "");break;}}}fclose(f);}- } else if (IsXnu()) {+ }++ if (IsXnu()) {char cpu_name[128] = {0};size_t size = sizeof(cpu_name);if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1) {id = cpu_name;}}- // TODO: other platforms++ std::string march;+#ifdef __x86_64__+ if (__cpu_march(__cpu_model.__cpu_subtype))+ march = __cpu_march(__cpu_model.__cpu_subtype);+#else+ long hwcap = getauxval(AT_HWCAP);+ if (hwcap & HWCAP_ASIMDHP)+ march += "+fp16";+ if (hwcap & HWCAP_ASIMDDP)+ march += "+dotprod";+#endif++ if (!march.empty()) {+ bool empty = id.empty();+ if (!empty)+ id += " (";+ id += march;+ if (!empty)+ id += ")";+ }+return id;}@@ -1034,7 +1073,7 @@ struct markdown_printer : public printer {return 15; // [jart]}if (field == "cpu_info") {- return 48; // [jart]+ return test::cpu_info.size(); // [jart]}if (field == "model_filename") {return 40; // [jart]@@ -1157,7 +1196,7 @@ struct markdown_printer : public printer {std::string value;char buf[128];if (field == "model") {- value = t.model_type;+ value = t.model_type;} else if (field == "size") {if (t.model_size < 1024*1024*1024) {snprintf(buf, sizeof(buf), "%.2f MiB", t.model_size / 1024.0 / 1024.0);@@ -1188,7 +1227,7 @@ struct markdown_printer : public printer {snprintf(buf, sizeof(buf), "%.2f", t.avg_ts());value = buf;} else if (vmap.find(field) != vmap.end()) {- value = vmap.at(field);+ value = replaceAll(replaceAll(vmap.at(field), ".gguf", ""), ".llamafile", ""); // [jart]} else {assert(false);exit(1);diff --git a/llamafile/govern.cpp b/llamafile/govern.cppindex 01c3f3ca..59a75553 100644--- a/llamafile/govern.cpp+++ b/llamafile/govern.cpp@@ -39,7 +39,7 @@// threshold, which jart configures to 80°C. the actual path of your cpu// temperature sensor may vary.-int llamafile_read_temperature(void) {+static int llamafile_read_temperature(void) {int fd;const char *path;char buf[13] = {0};@@ -54,7 +54,7 @@ int llamafile_read_temperature(void) {return atoi(buf);}-int llamafile_govern_threshold(void) {+static int llamafile_govern_threshold(void) {int temp;const char *s;if (!(s = getenv("LLAMAFILE_TEMPERATURE_MAX")))