Skip to content

Distribute and run LLMs with a single file.

Make some more benchmark tool fixes

Justine Tunneye67571914779
diff --git a/README.md b/README.md
index 54db8942..4319025a 100644
--- a/README.md
+++ b/README.md
@@ -175,7 +175,7 @@ try out llamafile with different kinds of LLMs.
| Mistral-7B-Instruct | 3.85 GB | [Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) | [mistral-7b-instruct-v0.2.Q4\_0.llamafile](https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q4_0.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile) |
| Phi-3-mini-4k-instruct | 7.67 GB | [Apache 2.0](https://huggingface.co/Mozilla/Phi-3-mini-4k-instruct-llamafile/blob/main/LICENSE) | [Phi-3-mini-4k-instruct.F16.llamafile](https://huggingface.co/Mozilla/Phi-3-mini-4k-instruct-llamafile/resolve/main/Phi-3-mini-4k-instruct.F16.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Phi-3-mini-4k-instruct-llamafile) |
| Mixtral-8x7B-Instruct | 30.03 GB | [Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) | [mixtral-8x7b-instruct-v0.1.Q5\_K\_M.llamafile](https://huggingface.co/Mozilla/Mixtral-8x7B-Instruct-v0.1-llamafile/resolve/main/mixtral-8x7b-instruct-v0.1.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Mixtral-8x7B-Instruct-v0.1-llamafile) |
-| WizardCoder-Python-34B | 22.23 GB | [LLaMA 2](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) | [wizardcoder-python-34b-v1.0.Q5\_K\_M.llamafile](https://huggingface.co/jartine/WizardCoder-Python-34B-V1.0-llamafile/resolve/main/wizardcoder-python-34b-v1.0.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/jartine/WizardCoder-Python-34B-V1.0-llamafile) |
+| WizardCoder-Python-34B | 22.23 GB | [LLaMA 2](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) | [wizardcoder-python-34b-v1.0.Q5\_K\_M.llamafile](https://huggingface.co/Mozilla/WizardCoder-Python-34B-V1.0-llamafile/resolve/main/wizardcoder-python-34b-v1.0.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/WizardCoder-Python-34B-V1.0-llamafile) |
| WizardCoder-Python-13B | 7.33 GB | [LLaMA 2](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) | [wizardcoder-python-13b.llamafile](https://huggingface.co/jartine/wizardcoder-13b-python/resolve/main/wizardcoder-python-13b.llamafile?download=true) | [See HF repo](https://huggingface.co/jartine/wizardcoder-13b-python) |
| LLaMA-3-Instruct-70B | 37.25 GB | [llama3](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/blob/main/Meta-Llama-3-Community-License-Agreement.txt) | [Meta-Llama-3-70B-Instruct.Q4\_0.llamafile](https://huggingface.co/Mozilla/Meta-Llama-3-70B-Instruct-llamafile/resolve/main/Meta-Llama-3-70B-Instruct.Q4_0.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Meta-Llama-3-70B-Instruct-llamafile) |
| LLaMA-3-Instruct-8B | 5.37 GB | [llama3](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/blob/main/Meta-Llama-3-Community-License-Agreement.txt) | [Meta-Llama-3-8B-Instruct.Q5\_K\_M.llamafile](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_M.llamafile?download=true) | [See HF repo](https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile) |
diff --git a/llama.cpp/llama-bench/llama-bench.cpp b/llama.cpp/llama-bench/llama-bench.cpp
index c335cb82..c436a431 100644
--- a/llama.cpp/llama-bench/llama-bench.cpp
+++ b/llama.cpp/llama-bench/llama-bench.cpp
@@ -19,6 +19,9 @@
#include <cosmo.h>
#include <libgen.h>
#include <sys/stat.h>
+#include <sys/auxv.h>
+#include <libc/intrin/x86.h>
+#include <libc/sysv/consts/hwcap.h>
#include "llama.cpp/ggml.h"
#include "llama.cpp/llama.h"
@@ -85,8 +88,18 @@ static T stdev(const std::vector<T> & v) {
return stdev;
}
-static std::string get_cpu_info() {
+static std::string replaceAll(std::string str, const std::string& from, const std::string& to) {
+ size_t start_pos = 0;
+ while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
+ str.replace(start_pos, from.length(), to);
+ start_pos += to.length(); // Handles case where 'to' is a substring of 'from'
+ }
+ return str;
+}
+
+static std::string get_cpu_info() { // [jart]
std::string id;
+
if (IsLinux()) {
FILE * f = fopen("/proc/cpuinfo", "r");
if (f) {
@@ -104,20 +117,46 @@ static std::string get_cpu_info() {
p[strlen(p) - 1] = '\0';
}
id = p;
+ id = replaceAll(id, " 96-Cores", "");
+ id = replaceAll(id, "(TM)", "");
+ id = replaceAll(id, "(R)", "");
break;
}
}
}
fclose(f);
}
- } else if (IsXnu()) {
+ }
+
+ if (IsXnu()) {
char cpu_name[128] = {0};
size_t size = sizeof(cpu_name);
if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1) {
id = cpu_name;
}
}
- // TODO: other platforms
+
+ std::string march;
+#ifdef __x86_64__
+ if (__cpu_march(__cpu_model.__cpu_subtype))
+ march = __cpu_march(__cpu_model.__cpu_subtype);
+#else
+ long hwcap = getauxval(AT_HWCAP);
+ if (hwcap & HWCAP_ASIMDHP)
+ march += "+fp16";
+ if (hwcap & HWCAP_ASIMDDP)
+ march += "+dotprod";
+#endif
+
+ if (!march.empty()) {
+ bool empty = id.empty();
+ if (!empty)
+ id += " (";
+ id += march;
+ if (!empty)
+ id += ")";
+ }
+
return id;
}
@@ -1034,7 +1073,7 @@ struct markdown_printer : public printer {
return 15; // [jart]
}
if (field == "cpu_info") {
- return 48; // [jart]
+ return test::cpu_info.size(); // [jart]
}
if (field == "model_filename") {
return 40; // [jart]
@@ -1157,7 +1196,7 @@ struct markdown_printer : public printer {
std::string value;
char buf[128];
if (field == "model") {
- value = t.model_type;
+ value = t.model_type;
} else if (field == "size") {
if (t.model_size < 1024*1024*1024) {
snprintf(buf, sizeof(buf), "%.2f MiB", t.model_size / 1024.0 / 1024.0);
@@ -1188,7 +1227,7 @@ struct markdown_printer : public printer {
snprintf(buf, sizeof(buf), "%.2f", t.avg_ts());
value = buf;
} else if (vmap.find(field) != vmap.end()) {
- value = vmap.at(field);
+ value = replaceAll(replaceAll(vmap.at(field), ".gguf", ""), ".llamafile", ""); // [jart]
} else {
assert(false);
exit(1);
diff --git a/llamafile/govern.cpp b/llamafile/govern.cpp
index 01c3f3ca..59a75553 100644
--- a/llamafile/govern.cpp
+++ b/llamafile/govern.cpp
@@ -39,7 +39,7 @@
// threshold, which jart configures to 80°C. the actual path of your cpu
// temperature sensor may vary.
-int llamafile_read_temperature(void) {
+static int llamafile_read_temperature(void) {
int fd;
const char *path;
char buf[13] = {0};
@@ -54,7 +54,7 @@ int llamafile_read_temperature(void) {
return atoi(buf);
}
-int llamafile_govern_threshold(void) {
+static int llamafile_govern_threshold(void) {
int temp;
const char *s;
if (!(s = getenv("LLAMAFILE_TEMPERATURE_MAX")))