Skip to content

Distribute and run LLMs with a single file.

Introduce LocalScore CLI

CJ Pais5748e861ae2a
diff --git a/Makefile b/Makefile
index de494c54..a11e1790 100644
--- a/Makefile
+++ b/Makefile
@@ -16,6 +16,7 @@ include llamafile/BUILD.mk
include llama.cpp/BUILD.mk
include stable-diffusion.cpp/BUILD.mk
include whisper.cpp/BUILD.mk
+include localscore/BUILD.mk
# the root package is `o//` by default
# building a package also builds its sub-packages
@@ -24,6 +25,7 @@ o/$(MODE)/: o/$(MODE)/llamafile \
o/$(MODE)/llama.cpp \
o/$(MODE)/stable-diffusion.cpp \
o/$(MODE)/whisper.cpp \
+ o/$(MODE)/localscore \
o/$(MODE)/third_party \
o/$(MODE)/depend.test
@@ -43,6 +45,7 @@ install: llamafile/zipalign.1 \
o/$(MODE)/llama.cpp/imatrix/imatrix \
o/$(MODE)/llama.cpp/quantize/quantize \
o/$(MODE)/llama.cpp/llama-bench/llama-bench \
+ o/$(MODE)/localscore/localscore \
o/$(MODE)/llama.cpp/perplexity/perplexity \
o/$(MODE)/llama.cpp/llava/llava-quantize \
o/$(MODE)/stable-diffusion.cpp/main \
@@ -55,6 +58,7 @@ install: llamafile/zipalign.1 \
$(INSTALL) o/$(MODE)/llama.cpp/imatrix/imatrix $(PREFIX)/bin/llamafile-imatrix
$(INSTALL) o/$(MODE)/llama.cpp/quantize/quantize $(PREFIX)/bin/llamafile-quantize
$(INSTALL) o/$(MODE)/llama.cpp/llama-bench/llama-bench $(PREFIX)/bin/llamafile-bench
+ $(INSTALL) o/$(MODE)/localscore/localscore $(PREFIX)/bin/localscore
$(INSTALL) build/llamafile-convert $(PREFIX)/bin/llamafile-convert
$(INSTALL) build/llamafile-upgrade-engine $(PREFIX)/bin/llamafile-upgrade-engine
$(INSTALL) o/$(MODE)/llama.cpp/perplexity/perplexity $(PREFIX)/bin/llamafile-perplexity
diff --git a/RELEASE.md b/RELEASE.md
new file mode 100644
index 00000000..76a1c5a1
--- /dev/null
+++ b/RELEASE.md
@@ -0,0 +1,129 @@
+# Making a Llamafile Release
+
+There are a few steps in making a Llamafile release which will be detailed in this document.
+
+The two primary artifacts of the release are the `llamafile-<version>.zip` and the binaries for the GitHub release.
+
+## Release Process
+
+Note: Step 2 and 3 are only needed if you are making a new release of the ggml-cuda.so and ggml-rocm.so shared libraries. You only need to do this when you are making changes to the CUDA code or the API's surrounding it. Otherwise you can use the previous release of the shared libraries.
+
+1. Update the version number in `version.h`
+2. Build the ggml-cuda.so and ggml-rocm.so shared libraries on Linux. You need to do this for Llamafile and LocalScore. Llamafile uses TINYBLAS as a default and LocalScore uses CUBLAS as a default for CUDA.
+ - For Llamafile you can do this by running the script `./llamafile/cuda.sh` and `./llamafile/rocm.sh` respectively.
+ - For LocalScore you can do this by running the script `./localscore/cuda.sh`.
+ - The files will be built and placed your home directory.
+3. Build the ggml-cuda.dll and ggml-rocm.dll shared libraries on Windows. You need to do this for Llamafile and LocalScore.
+ - You can do this by running the script `./llamafile/cuda.bat` and `./llamafile/rocm.bat` respectively.
+ - For LocalScore you can do this by running the script `./localscore/cuda.bat`.
+ - The files will be built and placed in the `build/release` directory.
+4. Build the project with `make -j8`
+5. Install the built project to your /usr/local/bin directory with `sudo make install PREFIX=/usr/local`
+
+### Llamafile Release Zip
+
+The easiest way to create the release zip is to:
+
+`make install PREFIX=<preferred_dir>/llamafile-<version>`
+
+After the directory is created, you will want to bundle the built shared libraries into the following release binaries:
+
+- `llamafile`
+- `localscore`
+- `whisperfile`
+
+You can do this for each binary with a command like the following:
+
+Note: You MUST put the shared libraries in the same directory as the binary you are creating.
+
+For llamafile and whisperfile you can do the following:
+
+`zipalign -j0 llamafile ggml-cuda.so ggml-rocm.so ggml-cuda.dll ggml-rocm.dll`
+`zipalign -j0 whisperfile ggml-cuda.so ggml-rocm.so ggml-cuda.dll ggml-rocm.dll`
+
+After doing this, delete the ggml-cuda.so and ggml-cuda.dll files from the directory, and copy + rename the ggml-cuda.localscore.so and ggml-cuda.localscore.dll files to the directory.
+
+```
+rm <path_to>/llamafile-<version>/bin/ggml-cuda.so <path_to>/llamafile-<version>/bin/ggml-cuda.dll
+cp ~/ggml-cuda.localscore.so <path_to>/llamafile-<version>/bin/ggml-cuda.so
+cp ~/ggml-cuda.localscore.dll <path_to>/llamafile-<version>/bin/ggml-cuda.dll
+```
+
+For localscore you can now package it:
+
+`zipalign -j0 localscore ggml-cuda.so ggml-rocm.so ggml-cuda.dll ggml-rocm.dll`
+
+After you have done this for all the binaries, you will want to get the existing PDFs (from the prior release) and add them to the directory:
+
+`cp <path_to>/doc/*.pdf <path_to>/llamafile-<version>/share/doc/llamafile/`
+
+The zip is structured as follows.
+
+```
+llamafile-<version>
+|-- README.md
+|-- bin
+| |-- llamafile
+| |-- llamafile-bench
+| |-- llamafile-convert
+| |-- llamafile-imatrix
+| |-- llamafile-perplexity
+| |-- llamafile-quantize
+| |-- llamafile-tokenize
+| |-- llamafile-upgrade-engine
+| |-- llamafiler
+| |-- llava-quantize
+| |-- localscore
+| |-- sdfile
+| |-- whisperfile
+| `-- zipalign
+`-- share
+ |-- doc
+ | `-- llamafile
+ | |-- llamafile-imatrix.pdf
+ | |-- llamafile-perplexity.pdf
+ | |-- llamafile-quantize.pdf
+ | |-- llamafile.pdf
+ | |-- llamafiler.pdf
+ | |-- llava-quantize.pdf
+ | |-- whisperfile.pdf
+ | `-- zipalign.pdf
+ `-- man
+ `-- man1
+ |-- llamafile-imatrix.1
+ |-- llamafile-perplexity.1
+ |-- llamafile-quantize.1
+ |-- llamafile.1
+ |-- llamafiler.1
+ |-- llava-quantize.1
+ |-- whisperfile.1
+ `-- zipalign.1
+```
+
+Before you zip the directory, you will want to remove the shared libraries from the directory.
+
+`rm *.so *.dll`
+
+You can zip the directory with the following command:
+
+`zip -r llamafile-<version>.zip llamafile-<version>`
+
+### Llamafile Release Binaries
+
+After you have built the zip it is quite easy to create the release binaries.
+
+The following binaries are part of the release:
+
+- `llamafile`
+- `llamafile-bench`
+- `llamafiler`
+- `sdfile`
+- `localscore`
+- `whisperfile`
+- `zipalign`
+
+You can use the script to create the appropriately named binaries:
+
+`./llamafile/release.sh -v <version> -s <source_dir> -d <dest_dir>`
+
+Make sure to move the llamafile-<version>.zip file to the <dest_dir> as well, and you are good to release after you've tested.
\ No newline at end of file
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
index 9c2f7f40..86ed4582 100644
--- a/llama.cpp/BUILD.mk
+++ b/llama.cpp/BUILD.mk
@@ -93,4 +93,4 @@ o/$(MODE)/llama.cpp: \
o/$(MODE)/llama.cpp/imatrix \
o/$(MODE)/llama.cpp/quantize \
o/$(MODE)/llama.cpp/perplexity \
- o/$(MODE)/llama.cpp/llama-bench
+ o/$(MODE)/llama.cpp/llama-bench \
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index a332617e..4a810580 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -17266,6 +17266,18 @@ GGML_CALL int ggml_backend_cuda_get_device_count() {
return ggml_cuda_info().device_count;
}
+GGML_CALL void ggml_backend_cuda_get_device_properties(int device, struct ggml_cuda_device_properties * properties) {
+ cudaDeviceProp prop;
+ CUDA_CHECK(cudaGetDeviceProperties(&prop, device));
+
+ strncpy(properties->name, prop.name, sizeof(properties->name) - 1);
+ properties->totalGlobalMem = prop.totalGlobalMem;
+ properties->multiProcessorCount = prop.multiProcessorCount;
+ properties->major = prop.major;
+ properties->minor = prop.minor;
+ snprintf(properties->compute, sizeof(properties->compute), "%d.%d", prop.major, prop.minor);
+}
+
GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) {
cudaDeviceProp prop;
CUDA_CHECK(cudaGetDeviceProperties(&prop, device));
diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.h
index b43050f6..0f5b70b6 100644
--- a/llama.cpp/ggml-cuda.h
+++ b/llama.cpp/ggml-cuda.h
@@ -20,6 +20,15 @@ extern "C" {
#define GGML_CUDA_MAX_DEVICES 16
+struct ggml_cuda_device_properties {
+ char name[256];
+ size_t totalGlobalMem;
+ int multiProcessorCount;
+ int major;
+ int minor;
+ char compute[8];
+};
+
GGML_API GGML_CALL bool ggml_cuda_link(const struct ggml_backend_api * backend_api);
// backend API
@@ -37,6 +46,7 @@ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_typ
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void);
+GGML_API GGML_CALL void ggml_backend_cuda_get_device_properties(int device, struct ggml_cuda_device_properties * properties);
GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
diff --git a/llama.cpp/ggml-metal.h b/llama.cpp/ggml-metal.h
index 4377cf8f..4448163f 100644
--- a/llama.cpp/ggml-metal.h
+++ b/llama.cpp/ggml-metal.h
@@ -35,6 +35,15 @@ struct ggml_cgraph;
extern "C" {
#endif
+struct ggml_metal_device_properties {
+ char name[256];
+ float memory;
+ int core_count;
+ int metal_version;
+ int gpu_family;
+ int gpu_family_common;
+};
+
void ggml_metal_link(const struct ggml_backend_api *);
//
@@ -56,6 +65,10 @@ GGML_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);
+GGML_API void ggml_backend_metal_get_device_properties(ggml_backend_t backend, struct ggml_metal_device_properties * properties);
+
+GGML_API void ggml_backend_metal_get_device_memory_usage(ggml_backend_t backend, float * used, float * total);
+
// helper to check if the device supports a specific family
// ideally, the user code should be doing these checks
// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.m
index 2aa77918..0ef8c86a 100644
--- a/llama.cpp/ggml-metal.m
+++ b/llama.cpp/ggml-metal.m
@@ -304,6 +304,10 @@ struct ggml_backend_metal_context {
id<MTLDevice> device;
id<MTLCommandQueue> queue;
+ int family;
+ int family_common;
+ int metal_version;
+
dispatch_queue_t d_queue;
struct ggml_metal_kernel kernels[GGML_METAL_KERNEL_TYPE_COUNT];
@@ -507,6 +511,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(int n_cb) {
for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
if ([ctx->device supportsFamily:i]) {
GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);
+ ctx->family = i - MTLGPUFamilyApple1 + 1;
break;
}
}
@@ -514,6 +519,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(int n_cb) {
for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
if ([ctx->device supportsFamily:i]) {
GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
+ ctx->family_common = i - MTLGPUFamilyCommon1 + 1;
break;
}
}
@@ -521,6 +527,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(int n_cb) {
for (int i = MTLGPUFamilyMetal3 + 5; i >= MTLGPUFamilyMetal3; --i) {
if ([ctx->device supportsFamily:i]) {
GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3 + 3, i);
+ ctx->metal_version = i - MTLGPUFamilyMetal3 + 3;
break;
}
}
@@ -3367,6 +3374,27 @@ void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_ca
ctx->abort_callback_data = user_data;
}
+void ggml_backend_metal_get_device_properties(ggml_backend_t backend, struct ggml_metal_device_properties * properties) {
+ GGML_ASSERT(ggml_backend_is_metal(backend));
+
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ strncpy(properties->name, [ctx->device name].UTF8String, sizeof(ctx->device.name));
+ properties->memory = ctx->device.recommendedMaxWorkingSetSize / 1073741824.0 ; // TODO is this what i want? mb? 1024*1024*1024 is bytes to gb
+ properties->gpu_family = ctx->family;
+ properties->gpu_family_common = ctx->family_common;
+ properties->metal_version = ctx->metal_version;
+}
+
+void ggml_backend_metal_get_device_memory_usage(ggml_backend_t backend, float * used, float * total) {
+ GGML_ASSERT(ggml_backend_is_metal(backend));
+
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ *used = (float)ctx->device.currentAllocatedSize / 1024.0 / 1024.0;
+ *total = (float)ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0;
+}
+
bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
GGML_ASSERT(ggml_backend_is_metal(backend));
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index b8ed12d0..1499a827 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -17565,6 +17565,10 @@ enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx) {
return ctx->cparams.pooling_type;
}
+int32_t llama_model_quant_str(const struct llama_model * model, char * buf, size_t buf_size) {
+ return snprintf(buf, buf_size, "%s", llama_model_ftype_name(model->ftype).c_str());
+}
+
int32_t llama_n_vocab(const struct llama_model * model) {
return model->hparams.n_vocab;
}
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index 181fca09..ea602059 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -478,6 +478,9 @@ extern "C" {
// Get the model's RoPE frequency scaling factor
LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);
+ // Get the quantization of the model
+ LLAMA_API int32_t llama_model_quant_str(const struct llama_model * model, char * buf, size_t buf_size);
+
// Functions to access the model's GGUF metadata scalar values
// - The functions return the length of the string on success, or -1 on failure
// - The output string is always null-terminated and cleared on failure
diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mk
index f47b3236..f2015502 100644
--- a/llama.cpp/main/BUILD.mk
+++ b/llama.cpp/main/BUILD.mk
@@ -13,6 +13,8 @@ o/$(MODE)/llama.cpp/main/main: \
o/$(MODE)/llama.cpp/main/embedding.o \
o/$(MODE)/llamafile/server/server.a \
o/$(MODE)/llama.cpp/server/server.a \
+ o/$(MODE)/localscore/localscore.a \
+ o/$(MODE)/third_party/mbedtls/mbedtls.a \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
o/$(MODE)/llamafile/highlight/highlight.a \
@@ -21,6 +23,7 @@ o/$(MODE)/llama.cpp/main/main: \
o/$(MODE)/llamafile/server/main.1.asc.zip.o \
$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o) \
$(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o) \
+ $(THIRD_PARTY_MBEDTLS_A_CERTS:%=o/$(MODE)/%.zip.o) \
$(LLAMA_CPP_MAIN_OBJS): llama.cpp/main/BUILD.mk
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index b6099dc1..5a4daabc 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -27,6 +27,7 @@
#include "llama.cpp/ggml-metal.h"
#include "llama.cpp/llava/llava.h"
#include "llama.cpp/server/server.h"
+#include "localscore/localscore.h"
#include "llamafile/server/prog.h"
static llama_context ** g_ctx;
@@ -152,6 +153,7 @@ enum Program {
CHATBOT,
EMBEDDING,
LLAMAFILER,
+ LOCALSCORE
};
enum Program determine_program(char *argv[]) {
@@ -168,6 +170,8 @@ enum Program determine_program(char *argv[]) {
prog = EMBEDDING;
} else if (!strcmp(argv[i], "--v2")) {
v2 = true;
+ } else if (!strcmp(argv[i], "--localscore")) {
+ prog = LOCALSCORE;
}
}
if (prog == SERVER && v2) {
@@ -221,6 +225,10 @@ int main(int argc, char ** argv) {
return embedding_cli(argc, argv);
}
+ if (prog == LOCALSCORE) {
+ return localscore_cli(argc, argv);
+ }
+
launch_sigint_thread();
gpt_params params;
diff --git a/llamafile/cuda.bat b/llamafile/cuda.bat
index faa30be2..4ff5d949 100644
--- a/llamafile/cuda.bat
+++ b/llamafile/cuda.bat
@@ -4,6 +4,24 @@
:: NVCUDA DLLs are provided by the installation of the windows GPU
:: driver on a Windows system that has a CUDA-capable GPU installed.
+mkdir build\release 2>nul
+
+for %%f in (
+ llama.cpp\ggml-cuda.cu
+ llama.cpp\ggml-cuda.h
+ llama.cpp\ggml-impl.h
+ llama.cpp\ggml-alloc.h
+ llama.cpp\ggml-common.h
+ llama.cpp\ggml-backend.h
+ llama.cpp\ggml-backend-impl.h
+ llama.cpp\ggml.h
+ llamafile\tinyblas.h
+ llamafile\tinyblas.cu
+ llamafile\llamafile.h
+) do copy %%f build\release
+
+cd build\release
+
nvcc --shared ^
--use_fast_math ^
-gencode arch=compute_60,code=sm_60 ^
diff --git a/llamafile/cuda.c b/llamafile/cuda.c
index 2d5df274..dc37429f 100644
--- a/llamafile/cuda.c
+++ b/llamafile/cuda.c
@@ -111,6 +111,7 @@ static struct Cuda {
typeof(ggml_backend_cuda_init) *GGML_CALL backend_init;
typeof(ggml_backend_cuda_split_buffer_type) *GGML_CALL split_buffer_type;
typeof(ggml_backend_cuda_reg_devices) *GGML_CALL reg_devices;
+ typeof(ggml_backend_cuda_get_device_properties) *GGML_CALL get_device_properties;
typeof(ggml_backend_cuda_get_device_memory) *GGML_CALL get_device_memory;
typeof(ggml_backend_cuda_get_device_count) *GGML_CALL get_device_count;
typeof(ggml_backend_cuda_unregister_host_buffer) *GGML_CALL unreg_host_buf;
@@ -728,6 +729,7 @@ static bool link_cuda_dso(const char *dso, const char *dir) {
ok &= !!(ggml_cuda.backend_init = imp(lib, "ggml_backend_cuda_init"));
ok &= !!(ggml_cuda.split_buffer_type = imp(lib, "ggml_backend_cuda_split_buffer_type"));
ok &= !!(ggml_cuda.reg_devices = imp(lib, "ggml_backend_cuda_reg_devices"));
+ ok &= !!(ggml_cuda.get_device_properties= imp(lib, "ggml_backend_cuda_get_device_properties"));
ok &= !!(ggml_cuda.get_device_memory = imp(lib, "ggml_backend_cuda_get_device_memory"));
ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_backend_cuda_get_device_count"));
ok &= !!(ggml_cuda.unreg_host_buf = imp(lib, "ggml_backend_cuda_unregister_host_buffer"));
@@ -1009,6 +1011,12 @@ GGML_CALL int ggml_backend_cuda_reg_devices(void) {
return ggml_cuda.reg_devices();
}
+GGML_CALL void ggml_backend_cuda_get_device_properties(int device, struct ggml_cuda_device_properties * properties) {
+ if (!llamafile_has_cuda())
+ return;
+ return ggml_cuda.get_device_properties(device, properties);
+}
+
GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t *free, size_t *total) {
if (!llamafile_has_cuda())
return;
diff --git a/llamafile/metal.c b/llamafile/metal.c
index 4e59c587..2bdb1fa4 100644
--- a/llamafile/metal.c
+++ b/llamafile/metal.c
@@ -74,6 +74,8 @@ static struct Metal {
typeof(ggml_backend_metal_set_n_cb) *backend_set_n_cb;
typeof(ggml_backend_metal_log_set_callback) *log_set_callback;
typeof(ggml_backend_reg_metal_init) *reg_init;
+ typeof(ggml_backend_metal_get_device_properties) *get_device_properties;
+ typeof(ggml_backend_metal_get_device_memory_usage) *get_device_memory_usage;
typeof(ggml_backend_metal_supports_family) *supports_family;
} ggml_metal;
@@ -218,6 +220,8 @@ static bool LinkMetal(const char *dso) {
ok &= !!(ggml_metal.backend_set_n_cb = cosmo_dlsym(lib, "ggml_backend_metal_set_n_cb"));
ok &= !!(ggml_metal.log_set_callback = cosmo_dlsym(lib, "ggml_backend_metal_log_set_callback"));
ok &= !!(ggml_metal.reg_init = cosmo_dlsym(lib, "ggml_backend_reg_metal_init"));
+ ok &= !!(ggml_metal.get_device_properties = cosmo_dlsym(lib, "ggml_backend_metal_get_device_properties"));
+ ok &= !!(ggml_metal.get_device_memory_usage = cosmo_dlsym(lib, "ggml_backend_metal_get_device_memory_usage"));
ok &= !!(ggml_metal.supports_family = cosmo_dlsym(lib, "ggml_backend_metal_supports_family"));
if (!ok) {
tinylog(Dlerror(), ": not all symbols could be imported\n", NULL);
@@ -323,6 +327,18 @@ ggml_backend_t ggml_backend_reg_metal_init(const char *params, void *user_data)
return ggml_metal.reg_init(params, user_data);
}
+void ggml_backend_metal_get_device_properties(ggml_backend_t backend, struct ggml_metal_device_properties *properties) {
+ if (!llamafile_has_metal())
+ return;
+ return ggml_metal.get_device_properties(backend, properties);
+}
+
+void ggml_backend_metal_get_device_memory_usage(ggml_backend_t backend, float *used, float *total) {
+ if (!llamafile_has_metal())
+ return;
+ return ggml_metal.get_device_memory_usage(backend, used, total);
+}
+
bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
if (!llamafile_has_metal())
return 0;
diff --git a/llamafile/release.sh b/llamafile/release.sh
new file mode 100755
index 00000000..5c9c1737
--- /dev/null
+++ b/llamafile/release.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Default values
+VERSION=""
+SOURCE_DIR=""
+DEST_DIR=""
+
+# Function to display help
+show_help() {
+ echo "Usage: $0 [OPTIONS]"
+ echo
+ echo "Script to copy and rename binary files with version numbers."
+ echo
+ echo "Options:"
+ echo " -v, --version VERSION Version number to append to filenames (required)"
+ echo " -s, --source DIR Source directory containing the binaries (required)"
+ echo " -d, --dest DIR Destination directory for renamed binaries (required)"
+ echo " -h, --help Display this help message and exit"
+ echo
+ echo "Example:"
+ echo " $0 -v 0.9.2 -s /llamafile-release/0.9.2/llamafile-0.9.2/bin -d /llamafile-release/0.9.2/release"
+ echo
+}
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+ case $1 in
+ -v|--version)
+ VERSION="$2"
+ shift 2
+ ;;
+ -s|--source)
+ SOURCE_DIR="$2"
+ shift 2
+ ;;
+ -d|--dest)
+ DEST_DIR="$2"
+ shift 2
+ ;;
+ -h|--help)
+ show_help
+ exit 0
+ ;;
+ *)
+ echo "Error: Unknown option: $1"
+ show_help
+ exit 1
+ ;;
+ esac
+done
+
+# Check if required arguments are provided
+if [ -z "$VERSION" ] || [ -z "$SOURCE_DIR" ] || [ -z "$DEST_DIR" ]; then
+ echo "Error: Missing required arguments"
+ show_help
+ exit 1
+fi
+
+# Check if source directory exists
+if [ ! -d "$SOURCE_DIR" ]; then
+ echo "Error: Source directory does not exist: $SOURCE_DIR"
+ exit 1
+fi
+
+# Create destination directory if it doesn't exist
+mkdir -p "$DEST_DIR"
+
+# List of binaries to copy and rename
+BINARIES=(
+ "llamafile"
+ "llamafile-bench"
+ "llamafiler"
+ "sdfile"
+ "localscore"
+ "whisperfile"
+ "zipalign"
+)
+
+# Copy and rename each binary
+for binary in "${BINARIES[@]}"; do
+ if [ -f "${SOURCE_DIR}/${binary}" ]; then
+ cp "${SOURCE_DIR}/${binary}" "${DEST_DIR}/${binary}-${VERSION}"
+ echo "Copied ${binary} to ${DEST_DIR}/${binary}-${VERSION}"
+ else
+ echo "Warning: ${SOURCE_DIR}/${binary} not found"
+ fi
+done
+
+# Make all binaries in destination directory executable
+chmod +x "${DEST_DIR}"/*
\ No newline at end of file
diff --git a/llamafile/rocm.bat b/llamafile/rocm.bat
index 114b9af7..6fa7d1a0 100644
--- a/llamafile/rocm.bat
+++ b/llamafile/rocm.bat
@@ -19,6 +19,24 @@
::
:: TODO(jart): How do we get this to not depend on VCRUNTIME140?
+mkdir build\release 2>nul
+
+for %%f in (
+ llama.cpp\ggml-cuda.cu
+ llama.cpp\ggml-cuda.h
+ llama.cpp\ggml-impl.h
+ llama.cpp\ggml-alloc.h
+ llama.cpp\ggml-common.h
+ llama.cpp\ggml-backend.h
+ llama.cpp\ggml-backend-impl.h
+ llama.cpp\ggml.h
+ llamafile\tinyblas.h
+ llamafile\tinyblas.cu
+ llamafile\llamafile.h
+) do copy %%f build\release
+
+cd build\release
+
"%HIP_PATH_57%\bin\clang++.exe" ^
-fuse-ld=lld ^
-shared ^
diff --git a/localscore/BUILD.mk b/localscore/BUILD.mk
new file mode 100644
index 00000000..7d36b528
--- /dev/null
+++ b/localscore/BUILD.mk
@@ -0,0 +1,41 @@
+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐
+#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘
+
+PKGS += LOCALSCORE
+
+LOCALSCORE_FILES := $(wildcard localscore/*.*)
+LOCALSCORE_INCS = $(filter %.inc,$(LOCALSCORE_FILES))
+LOCALSCORE_SRCS_C = $(filter %.c,$(LOCALSCORE_FILES))
+LOCALSCORE_SRCS_CPP = $(filter %.cpp,$(LOCALSCORE_FILES))
+LOCALSCORE_SRCS = $(LOCALSCORE_SRCS_C) $(LOCALSCORE_SRCS_CPP)
+
+LOCALSCORE_HDRS = \
+ $(filter %.h,$(LOCALSCORE_FILES)) \
+ $(filter %.hpp,$(LOCALSCORE_FILES))
+
+LOCALSCORE_OBJS = \
+ $(LOCALSCORE_SRCS_C:%.c=o/$(MODE)/%.o) \
+ $(LOCALSCORE_SRCS_CPP:%.cpp=o/$(MODE)/%.o)
+
+# Build the library
+o/$(MODE)/localscore/localscore.a: \
+ $(LOCALSCORE_OBJS) \
+
+# Any specific compiler flags needed
+$(LOCALSCORE_OBJS): private \
+ CXXFLAGS += \
+ -frtti \
+ -Wno-deprecated-declarations
+
+# Main executable with mbedtls dependency
+o/$(MODE)/localscore/localscore: \
+ o/$(MODE)/localscore/main.o \
+ o/$(MODE)/localscore/localscore.a \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
+ o/$(MODE)/third_party/mbedtls/mbedtls.a \
+
+$(LOCALSCORE_OBJS): localscore/BUILD.mk
+
+.PHONY: o/$(MODE)/localscore
+o/$(MODE)/localscore: \
+ o/$(MODE)/localscore/localscore \
diff --git a/localscore/README.md b/localscore/README.md
new file mode 100644
index 00000000..1fd1654a
--- /dev/null
+++ b/localscore/README.md
@@ -0,0 +1,187 @@
+# LocalScore
+
+LocalScore is an open-source benchmarking tool designed to measure how fast Large Language Models (LLMs) run on your specific hardware. It also provides a public database for benchmark results, helping you make informed decisions about running AI models locally.
+
+You can view the leaderboard at [localscore.ai](https://localscore.ai)
+
+![Example of the LocalScore CLI](localscore-example.jpeg)
+
+<p align="center">
+<i>
+<br />
+<a href="https://builders.mozilla.org/"><img src="../llamafile/mozilla-logo-bw-rgb.png" width="200"></a><br/>
+<a href="https://localscore.ai">LocalScore</a> is a <a href="https://builders.mozilla.org/">Mozilla Builders</a> project.
+</i>
+</p>
+
+## Features
+
+- **Comprehensive Performance Metrics**: Measures prompt processing speed, generation speed, and time to first token
+- **Hardware Support**: Works with CPUs, NVIDIA GPUs, AMD GPUs, and Apple Silicon
+- **Result Sharing**: Optional submission of benchmark results to a public database at [localscore.ai](https://localscore.ai)
+- **Cross-Platform**: Works on Windows, macOS, and Linux
+
+## What LocalScore Measures
+
+LocalScore evaluates three key performance metrics:
+
+1. **Prompt Processing Speed**: How quickly your system processes input text (tokens per second)
+2. **Generation Speed**: How fast your system generates new text (tokens per second)
+3. **Time to First Token**: The latency before the first response appears (milliseconds)
+
+These metrics are combined into a single **LocalScore** value using a geometric mean:
+
+$`\text{score} = 10 \cdot \sqrt[3]{\text{avg\_prompt\_tps} \cdot \text{avg\_gen\_tps} \cdot \frac{1000}{\text{avg\_ttft\_ms}}}`$
+
+As a general guideline:
+- A score of 1,000+ is excellent
+- A score of 250 is acceptable to good for most people
+- A score of 100 is relatively poor
+
+## Running LocalScore
+
+There are four primary ways to run LocalScore. The first two are covered on the [LocalScore download page](https://localscore.ai/download).
+
+In addition to the following instructions, you can always build from source. For that case please follow the [Llamafile build instructions](https://github.com/Mozilla-Ocho/llamafile).
+
+### Option 1: Download and Run a LocalScore Bundle
+
+The LocalScore bundle include the LocalScore binary and a model inside of it.
+
+Visit [localscore.ai](https://localscore.ai/download) to get the current bundles.
+
+### Option 2: Download the LocalScore release directly
+
+Download the appropriate binary for your operating system:
+[Latest Release Download Page](https://github.com/Mozilla-Ocho/llamafile/releases/latest)
+
+#### MacOS/Linux
+```bash
+chmod +x localscore
+./localscore -m path/to/model.gguf
+```
+
+#### Windows
+```powershell
+localscore.exe -m path\to\model.gguf
+```
+
+### Option 3: Download a Llamafile bundle
+
+Every new llamafile (>=v0.9.2) contains commands to run LocalScore.
+
+Example:
+
+#### MacOS/Linux
+```bash
+# Download a llamafile from Hugging Face
+curl -O https://huggingface.co/Mozilla/Llama-3.2-1B-Instruct-llamafile/resolve/main/Llama-3.2-1B-Instruct.Q4_K_M.llamafile
+
+# Run LocalScore
+chmod +x Llama-3.2-1B-Instruct.Q4_K_M.llamafile
+./Llama-3.2-1B-Instruct.Q4_K_M.llamafile --bench
+```
+
+#### Windows
+Download any llamafile smaller than 4GB from Hugging Face and run it.
+```powershell
+Llama-3.2-1B-Instruct.Q4_K_M.llamafile.exe --bench
+```
+
+### Option 4: Run LocalScore from Llamafile
+
+If you have Llamafile installed, you can run LocalScore directly from it.
+
+#### MacOS/Linux
+
+```bash
+# Run LocalScore
+llamafile --bench -m path/to/model.gguf
+```
+
+#### Windows
+
+```powershell
+llamafile.exe --bench -m path\to\model.gguf
+```
+
+### Command Line Options
+
+```
+usage: localscore [options]
+
+options:
+ -h, --help Show this help message
+ -m, --model <filename> Model to benchmark (default: path/to/default)
+ -c, --cpu Disable GPU acceleration (alias for --gpu=disabled)
+ -g, --gpu <auto|amd|apple|nvidia|disabled> GPU backend to use (default: "auto")
+ -i, --gpu-index <i> Select GPU by index (default: 0)
+ --list-gpus List available GPUs and exit
+ -o, --output <csv|json|md> Output format (default: md)
+ -v, --verbose Enable verbose output
+ -y, --send-results Send results without confirmation
+ -n, --no-send-results Disable sending results
+ -e, --extended Run 4 repetitions (shortcut for --reps=4)
+ --long Run 16 repetitions (shortcut for --reps=16)
+ --reps <N> Set custom number of repetitions
+```
+
+#### Running with CPU Only
+
+```bash
+./localscore -m path/to/model.gguf --cpu
+```
+
+#### Send Results Automatically
+
+```bash
+./localscore -m path/to/model.gguf -y
+```
+
+#### Run Each Test 4 Times
+
+```bash
+./localscore -m path/to/model.gguf -e
+```
+
+## Data Collection
+
+When you submit benchmark results to localscore.ai, the following non-personally identifiable system information is collected:
+
+- CPU model and configuration
+- GPU model and configuration
+- Operating system and version
+- RAM amount
+- Benchmark performance metrics
+
+This data helps build a comprehensive database of hardware performance for LLM inference, allowing users to compare different setups and make informed decisions.
+
+## Limitations
+
+- **Windows File Size**: Due to Windows limitations, llamafile's larger than 4GB will not run directly. Windows users should use LocalScore as a standalone utility and pass in models in GGUF format.
+- **Single-GPU Focus**: Currently, LocalScore only supports single-GPU setups, which represents the most practical approach for most users running LLMs locally.
+- **Early Development**: LocalScore is in relatively early development, so you may encounter occasional issues. Please file a GitHub issue if you encounter any problems.
+
+## Troubleshooting
+
+Check out the [Troubleshooting](doc/Troubleshooting.md) doc for common issues and solutions.
+
+## Contributing
+
+Contributions are welcome! See the main [Llamafile README](https://github.com/Mozilla-Ocho/llamafile) for building instructions and development guidelines.
+
+## Acknowledgements
+
+LocalScore was created with support from [Mozilla Builders](https://builders.mozilla.org/) and builds upon the excellent work of:
+- [llama.cpp](https://github.com/ggml-org/llama.cpp)
+- [Llamafile](https://github.com/Mozilla-Ocho/llamafile)
+
+## License
+
+LocalScore is released under the [MIT License](LICENSE).
+
+## Links
+
+- [Official Website](https://localscore.ai)
+- [GitHub Repository](localscore)
+- [Issue Tracker](https://github.com/Mozilla-Ocho/llamafile/issues)
\ No newline at end of file
diff --git a/localscore/apple.cpp b/localscore/apple.cpp
new file mode 100644
index 00000000..823d2e50
--- /dev/null
+++ b/localscore/apple.cpp
@@ -0,0 +1,174 @@
+#include <cosmo.h>
+#include <dlfcn.h>
+
+#include "apple.h"
+#include "llama.cpp/common.h"
+
+static void *imp(void *lib, const char *sym) {
+ void *fun = cosmo_dlsym(lib, sym);
+ if (!fun)
+ tinylog(__func__, ": error: failed to import symbol: ", sym, "\n", NULL);
+ return fun;
+}
+
+static struct IOReport {
+ CFDictionaryRef (*IOReportCopyChannelsInGroup)(CFStringRef, CFStringRef, uint64_t, uint64_t, uint64_t);
+ IOReportSubscriptionRef (*IOReportCreateSubscription)(CVoidRef, CFMutableDictionaryRef, CFMutableDictionaryRef*, uint64_t, CFTypeRef);
+ CFDictionaryRef (*IOReportCreateSamples)(IOReportSubscriptionRef, CFMutableDictionaryRef, CFTypeRef);
+ CFDictionaryRef (*IOReportCreateSamplesDelta)(CFDictionaryRef, CFDictionaryRef, CFTypeRef);
+ CFStringRef (*IOReportChannelGetChannelName)(CFDictionaryRef);
+ int64_t (*IOReportSimpleGetIntegerValue)(CFDictionaryRef, int32_t);
+ CFStringRef (*IOReportChannelGetUnitLabel)(CFDictionaryRef);
+} io_report;
+
+static struct CoreFoundation {
+ CFMutableDictionaryRef (*CFDictionaryCreateMutableCopy)(CVoidRef, long, CFDictionaryRef);
+ long (*CFDictionaryGetCount)(CFDictionaryRef);
+ void (*CFShow)(CFTypeRef);
+ CVoidRef (*CFDictionaryGetValue)(CFDictionaryRef, CVoidRef);
+ CFStringRef (*CFStringCreateWithCString)(CVoidRef, const char*, int);
+ void (*CFRelease)(CFTypeRef);
+ int (*CFArrayGetCount)(CFArrayRef);
+ CFTypeRef (*CFArrayGetValueAtIndex)(CFArrayRef, int);
+ CFArrayRef (*CFArrayCreateCopy)(CVoidRef, CFArrayRef);
+ bool (*CFStringGetCString)(CFStringRef, char *, int, int);
+} core_foundation;
+
+bool init_apple_mon() {
+ void *lib = cosmo_dlopen("/usr/lib/libIOReport.dylib", RTLD_LAZY);
+ if (!lib) {
+ tinylog(__func__, ": error: failed to open IOKit framework\n", NULL);
+ return false;
+ }
+
+ bool ok = true;
+
+ ok &= !!(io_report.IOReportCopyChannelsInGroup = (CFDictionaryRef (*)(CFStringRef, CFStringRef, uint64_t, uint64_t, uint64_t))imp(lib, "IOReportCopyChannelsInGroup"));
+ ok &= !!(io_report.IOReportCreateSubscription = (IOReportSubscriptionRef (*)(CVoidRef, CFMutableDictionaryRef, CFMutableDictionaryRef*, uint64_t, CFTypeRef))imp(lib, "IOReportCreateSubscription"));
+ ok &= !!(io_report.IOReportCreateSamples = (CFDictionaryRef (*)(IOReportSubscriptionRef, CFMutableDictionaryRef, CFTypeRef))imp(lib, "IOReportCreateSamples"));
+ ok &= !!(io_report.IOReportCreateSamplesDelta = (CFDictionaryRef (*)(CFDictionaryRef, CFDictionaryRef, CFTypeRef))imp(lib, "IOReportCreateSamplesDelta"));
+ ok &= !!(io_report.IOReportChannelGetChannelName = (CFStringRef (*)(CFDictionaryRef))imp(lib, "IOReportChannelGetChannelName"));
+ ok &= !!(io_report.IOReportSimpleGetIntegerValue = (int64_t (*)(CFDictionaryRef, int32_t))imp(lib, "IOReportSimpleGetIntegerValue"));
+ ok &= !!(io_report.IOReportChannelGetUnitLabel = (CFStringRef (*)(CFDictionaryRef))imp(lib, "IOReportChannelGetUnitLabel"));
+
+ if (!ok) {
+ tinylog(__func__, ": error: not all IOReport symbols could be imported\n", NULL);
+ cosmo_dlclose(lib);
+ return false;
+ }
+
+ ok &= !!(core_foundation.CFDictionaryCreateMutableCopy = (CFMutableDictionaryRef (*)(void*, long, CFDictionaryRef))imp(lib, "CFDictionaryCreateMutableCopy"));
+ ok &= !!(core_foundation.CFDictionaryGetCount = (long (*)(CFDictionaryRef))imp(lib, "CFDictionaryGetCount"));
+ ok &= !!(core_foundation.CFShow = (void (*)(CFTypeRef))imp(lib, "CFShow"));
+ ok &= !!(core_foundation.CFDictionaryGetValue = ( void* (*)(CFDictionaryRef, void*))imp(lib, "CFDictionaryGetValue"));
+ ok &= !!(core_foundation.CFStringCreateWithCString = (CFStringRef (*)(CVoidRef, const char*, int))imp(lib, "CFStringCreateWithCString"));
+ ok &= !!(core_foundation.CFRelease = (void (*)(CFTypeRef))imp(lib, "CFRelease"));
+ ok &= !!(core_foundation.CFArrayGetCount = (int (*)(CFArrayRef))imp(lib, "CFArrayGetCount"));
+ ok &= !!(core_foundation.CFArrayGetValueAtIndex = (CFTypeRef (*)(CFArrayRef, int))imp(lib, "CFArrayGetValueAtIndex"));
+ ok &= !!(core_foundation.CFArrayCreateCopy = (CFArrayRef (*)(CVoidRef, CFArrayRef))imp(lib, "CFArrayCreateCopy"));
+ ok &= !!(core_foundation.CFStringGetCString = (bool (*)(CFStringRef, char *, int, int))imp(lib, "CFStringGetCString"));
+
+ if (!ok) {
+ tinylog(__func__, ": error: not all CoreFoundation symbols could be imported\n", NULL);
+ cosmo_dlclose(lib);
+ return false;
+ }
+
+ return true;
+}
+
+static bool get_cstring_from_cfstring(CFStringRef cfString, char* buffer, size_t bufferSize) {
+ return core_foundation.CFStringGetCString(cfString, buffer, bufferSize, 0x08000100);
+}
+
+static char* get_unit_label(CFDictionaryRef item) {
+ static char unit[64];
+ CFStringRef u = io_report.IOReportChannelGetUnitLabel(item);
+ if (u) {
+ if (!get_cstring_from_cfstring(u, unit, sizeof(unit))) {
+ strcpy(unit, "Unknown");
+ }
+ core_foundation.CFRelease(u);
+ } else {
+ strcpy(unit, "N/A");
+ }
+ return unit;
+}
+
+static double get_item_energy_millijoules(CFDictionaryRef item, const char* name) {
+ char* unit = get_unit_label(item);
+ double energy = (double)io_report.IOReportSimpleGetIntegerValue(item, 0);
+ double energy_millijoules = 0;
+
+ if (strcmp(name, "CPU Energy") == 0 || strcmp(name, "GPU Energy") == 0 || strstr(name, "ANE") != NULL) {
+ if (strcmp(unit, "mJ") == 0) {
+ energy_millijoules = energy;
+ } else if (strcmp(unit, "uJ") == 0) {
+ energy_millijoules = energy / 1e3;
+ } else if (strcmp(unit, "nJ") == 0) {
+ energy_millijoules = energy / 1e6;
+ } else {
+ printf("Unknown unit: %s for channel: %s\n", unit, name);
+ return 0;
+ }
+ }
+
+ return energy_millijoules;
+}
+
+
+void am_release(void* obj) {
+ core_foundation.CFRelease(obj);
+}
+
+void am_print_object(CFTypeRef obj) {
+ core_foundation.CFShow(obj);
+}
+
+CFMutableDictionaryRef am_get_power_channels() {
+ CFStringRef energy_str = core_foundation.CFStringCreateWithCString(NULL, "Energy Model", 0x08000100);
+ CFDictionaryRef channels = io_report.IOReportCopyChannelsInGroup(energy_str, NULL, 0, 0, 0);
+ core_foundation.CFRelease(energy_str);
+
+ CFMutableDictionaryRef channels_mut = core_foundation.CFDictionaryCreateMutableCopy(NULL, core_foundation.CFDictionaryGetCount(channels), channels);
+ core_foundation.CFRelease(channels);
+
+ return channels_mut;
+}
+
+IOReportSubscriptionRef am_get_subscription(CFMutableDictionaryRef channels_mut) {
+ CFMutableDictionaryRef subscription;
+ IOReportSubscriptionRef s = io_report.IOReportCreateSubscription(NULL, channels_mut, &subscription, 0, NULL);
+ return s;
+}
+
+// TODO need some way of freeing the CFDictionaryRef?
+CFDictionaryRef am_sample_power(IOReportSubscriptionRef sub, CFMutableDictionaryRef channels) {
+ return io_report.IOReportCreateSamples(sub, channels, NULL);
+}
+
+double am_sample_to_millijoules(CFDictionaryRef sample) {
+ CFStringRef key = core_foundation.CFStringCreateWithCString(NULL, "IOReportChannels", 0x08000100);
+ CFArrayRef report = core_foundation.CFDictionaryGetValue(sample, key);
+ core_foundation.CFRelease(key);
+
+ CFIndex count = core_foundation.CFArrayGetCount(report);
+ double total_energy_millijoules = 0;
+
+ for (CFIndex i = 0; i < count; i++) {
+ CFDictionaryRef item = core_foundation.CFArrayGetValueAtIndex(report, i);
+ CFStringRef n = io_report.IOReportChannelGetChannelName(item);
+ char name[64] = {0};
+
+ if (!core_foundation.CFStringGetCString(n, name, sizeof(name), 0x08000100)) {
+ printf("Failed to get channel name\n");
+ core_foundation.CFRelease(n);
+ continue;
+ }
+
+ total_energy_millijoules += get_item_energy_millijoules(item, name);
+ core_foundation.CFRelease(n);
+ }
+
+ return total_energy_millijoules;
+}
\ No newline at end of file
diff --git a/localscore/apple.h b/localscore/apple.h
new file mode 100644
index 00000000..045064db
--- /dev/null
+++ b/localscore/apple.h
@@ -0,0 +1,22 @@
+typedef void* CFStringRef;
+typedef void* CFDictionaryRef;
+typedef void* CFMutableDictionaryRef;
+typedef void* CFTypeRef;
+typedef void* CFArrayRef;
+typedef void* IOReportSubscriptionRef;
+typedef void* CVoidRef;
+typedef int CFIndex;
+
+bool init_apple_mon();
+
+void am_release(void* obj);
+
+void am_print_object(CFTypeRef obj);
+
+CFMutableDictionaryRef am_get_power_channels();
+
+IOReportSubscriptionRef am_get_subscription(CFMutableDictionaryRef power_channel);
+
+CFDictionaryRef am_sample_power(IOReportSubscriptionRef sub, CFMutableDictionaryRef channels);
+
+double am_sample_to_millijoules(CFDictionaryRef sample);
\ No newline at end of file
diff --git a/localscore/ascii_digits.h b/localscore/ascii_digits.h
new file mode 100644
index 00000000..30316cbd
--- /dev/null
+++ b/localscore/ascii_digits.h
@@ -0,0 +1,175 @@
+#ifndef ASCII_DIGITS_H
+#define ASCII_DIGITS_H
+
+#include <string>
+#include <iostream>
+
+namespace ascii_display {
+
+// Array of ASCII art digits
+extern const char* digits[10];
+
+/**
+ * @brief Prints a number using ASCII art characters
+ * @param number The number to print
+ */
+void printLargeNumber(int number);
+
+inline void printLargeNumber(int number) {
+ // Convert number to string
+ std::string num_str = std::to_string(number);
+
+ // Handle negative numbers
+ bool is_negative = number < 0;
+ if (is_negative) {
+ num_str = num_str.substr(1); // Remove the minus sign for processing
+ }
+
+ // Print each row
+ for (int row = 0; row < 5; row++) {
+ for (char digit : num_str) {
+ int d = digit - '0'; // Convert char to int
+
+ // Find the start and end of the current row in the digit's string
+ const char* ptr = digits[d];
+ int line_start = 0;
+ for (int i = 0; i < row; i++) {
+ while (ptr[line_start] != '\n') line_start++;
+ line_start++; // Skip the newline
+ }
+
+ int line_end = line_start;
+ while (ptr[line_end] && ptr[line_end] != '\n') line_end++;
+
+ // Print the current row
+ for (int i = line_start; i < line_end; i++) {
+ std::cout << ptr[i];
+ }
+ std::cout << " "; // Space between digits
+ }
+ std::cout << "\n";
+ }
+}
+
+// inline void print_logo() {
+// std::cout <<
+// "██ ██████ ██████ █████ ██ \n"
+// "██ ██ ██ ██ ██ ██ ██ \n"
+// "██ ██ ██ ██ ███████ ██ \n"
+// "██ ██ ██ ██ ██ ██ ██ \n"
+// "███████ ██████ ██████ ██ ██ ███████ \n"
+// " \n"
+// "███████ ██████ ██████ ██████ ███████ \n"
+// "██ ██ ██ ██ ██ ██ ██ \n"
+// "███████ ██ ██ ██ ██████ █████ \n"
+// " ██ ██ ██ ██ ██ ██ ██ \n"
+// "███████ ██████ ██████ ██ ██ ███████ \n"
+// "\n"
+// "---------------------------------------- \n";
+// }
+
+
+
+// inline void print_logo() {
+// std::cout <<
+// " ___ ________ ________ ________ ___ \n"
+// "|\\ \\ |\\ __ \\|\\ ____\\|\\ __ \\|\\ \\ \n"
+// "\\ \\ \\ \\ \\ \\|\\ \\ \\ \\___|\\ \\|\\ \\ \\ \\ \n"
+// " \\ \\ \\ \\ \\ \\\\\\ \\ \\ \\ \\ \\ __ \\ \\ \\ \n"
+// " \\ \\ \\____\\ \\ \\\\\\ \\ \\ \\____\\ \\ \\ \\ \\ \\ \\____ \n"
+// " \\ \\_______\\ \\_______\\ \\_______\\ \\__\\ \\__\\ \\_______\\\n"
+// " \\|_______|\\|_______|\\|_______|\\|__|\\|__|\\|_______|\n"
+// " ________ ________ ________ ________ _______ \n"
+// "|\\ ____\\|\\ ____\\|\\ __ \\|\\ __ \\|\\ ___ \\ \n"
+// "\\ \\ \\___|\\ \\___|\\ \\|\\ \\ \\ \\|\\ \\ \\ __/| \n"
+// " \\ \\_____ \\ \\ \\ \\ \\ \\\\\\ \\ \\ _ _\\ \\ \\_|/__ \n"
+// " \\|____|\\ \\ \\ \\____\\ \\ \\\\\\ \\ \\ \\\\ \\\\ \\ \\_|\\ \\ \n"
+// " ____\\_\\ \\ \\_______\\ \\_______\\ \\__\\\\ _\\\\ \\_______\\\n"
+// " |\\_________\\|_______|\\|_______|\\|__|\\|__|\\|_______|\n"
+// " \\|_________| \n";
+// }
+
+inline void print_logo() {
+ std::cout << " __ ____ _________ __ _____ __________ ____ ______\n"
+ << " / / / __ \\/ ____/ | / / / ___// ____/ __ \\/ __ \\/ ____/\n"
+ << " / / / / / / / / /| | / / \\__ \\/ / / / / / /_/ / __/ \n"
+ << " / /___/ /_/ / /___/ ___ |/ /______/ / /___/ /_/ / _, _/ /___ \n"
+ << "/_____/\\____/\\____/_/ |_/_____/____/\\____/\\____/_/ |_/_____/ \n";
+}
+
+// Definition of the digits array
+inline const char* digits[10] = {
+ // 0
+ " ██████ \n"
+ "██ ██\n"
+ "██ ██\n"
+ "██ ██\n"
+ " ██████ \n",
+
+ // 1
+ " ██ \n"
+ "███ \n"
+ " ██ \n"
+ " ██ \n"
+ " ██ \n",
+
+ // 2
+ "██████ \n"
+ " ██ \n"
+ " █████ \n"
+ "██ \n"
+ "███████ \n",
+
+ // 3
+ "██████ \n"
+ " ██ \n"
+ " █████ \n"
+ " ██ \n"
+ "██████ \n",
+
+ // 4
+ "██ ██ \n"
+ "██ ██ \n"
+ "███████ \n"
+ " ██ \n"
+ " ██ \n",
+
+ // 5
+ "███████ \n"
+ "██ \n"
+ "██████ \n"
+ " ██ \n"
+ "██████ \n",
+
+ // 6
+ " ██████ \n"
+ "██ \n"
+ "███████ \n"
+ "██ ██\n"
+ " ██████ \n",
+
+ // 7
+ "███████ \n"
+ " ██ \n"
+ " ██ \n"
+ " ██ \n"
+ " ██ \n",
+
+ // 8
+ " █████ \n"
+ "██ ██ \n"
+ " █████ \n"
+ "██ ██ \n"
+ " █████ \n",
+
+ // 9
+ " ██████ \n"
+ "██ ██\n"
+ " ███████\n"
+ " ██\n"
+ " ██████ \n"
+};
+
+} // namespace ascii_display
+
+#endif // ASCII_DIGITS_H
diff --git a/localscore/benchmark.cpp b/localscore/benchmark.cpp
new file mode 100644
index 00000000..dac93781
--- /dev/null
+++ b/localscore/benchmark.cpp
@@ -0,0 +1,363 @@
+#include "benchmark.h"
+
+#include <iterator>
+
+#include "llama.cpp/ggml-cuda.h"
+#include "llama.cpp/string.h"
+#include "llamafile/string.h"
+#include "utils.h"
+
+test::test(const cmd_params &inst, const llama_model *lmodel,
+ llama_context *context, int repetitions, PowerSampler *sampler) {
+ model_filename = lf::basename(strdup(inst.model.c_str())); // [jart]
+ char buf[128];
+ llama_model_desc(lmodel, buf, sizeof(buf));
+ model_type = buf;
+ llama_model_meta_val_str(lmodel, "general.name", buf, sizeof(buf));
+ model_name = buf;
+ llama_model_quant_str(lmodel, buf, sizeof(buf));
+ model_quant_str = buf;
+ model_size = llama_model_size(lmodel);
+ model_n_params = llama_model_n_params(lmodel);
+ llama_model_meta_val_str(lmodel, "general.size_label", buf, sizeof(buf));
+ model_params_str = buf;
+ n_batch = inst.n_batch;
+ n_ubatch = inst.n_ubatch;
+ n_threads = inst.n_threads;
+ type_k = inst.type_k;
+ type_v = inst.type_v;
+ n_gpu_layers = inst.n_gpu_layers;
+ split_mode = inst.split_mode;
+ main_gpu = inst.main_gpu;
+ no_kv_offload = inst.no_kv_offload;
+ flash_attn = inst.flash_attn;
+ tensor_split = inst.tensor_split;
+ use_mmap = inst.use_mmap;
+ embeddings = inst.embeddings;
+ n_prompt = inst.n_prompt;
+ n_gen = inst.n_gen;
+ reps = repetitions;
+ test_completed = false;
+ curr_run = 0;
+ t_gen = 0;
+ t_processed = 0;
+ monitor_result = {0.0};
+ pwr_sampler = sampler;
+
+ if (n_prompt > 0 && n_gen == 0) {
+ snprintf(buf, sizeof(buf), "pp%d", n_prompt);
+ } else if (n_gen > 0 && n_prompt == 0) {
+ snprintf(buf, sizeof(buf), "tg%d", n_gen);
+ } else {
+ snprintf(buf, sizeof(buf), "pp%d+tg%d", n_prompt, n_gen);
+ }
+ name = buf;
+
+ // RFC 3339 date-time format
+ time_t t = time(NULL);
+ std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t));
+ test_time = buf;
+
+ ctx = context;
+}
+
+void test::run() {
+
+ // run the test for however many repetitions specified
+ pwr_sampler->start();
+ for (int i = 0; i < reps; i++) {
+ curr_run = i;
+ t_processed = 0;
+ t_gen = 0;
+ llama_kv_cache_clear(ctx);
+
+ time_interval interval;
+ interval.start = utils::get_time_ns();
+ interval.end = 0;
+ test_intervals.push_back(interval);
+
+ if (n_prompt > 0) {
+ test_prompt();
+ }
+ if (n_gen > 0) {
+ test_gen();
+ }
+
+ test_intervals.back().end = utils::get_time_ns();
+ }
+ monitor_result = pwr_sampler->stop();
+
+ test_completed = true;
+}
+
+void test::test_prompt() {
+ llama_set_n_threads(ctx, n_threads, n_threads);
+
+ const llama_model *model = llama_get_model(ctx);
+ const int32_t n_vocab = llama_n_vocab(model);
+
+ std::vector<llama_token> tokens(n_batch);
+
+ int n_processed = 0;
+
+ time_interval interval;
+ interval.start = utils::get_time_ns();
+ interval.end = 0;
+ prompt_intervals.push_back(interval);
+
+ while (n_processed < n_prompt) {
+ int n_tokens = std::min(n_prompt - n_processed, n_batch);
+ tokens[0] = n_processed == 0 && llama_add_bos_token(model)
+ ? llama_token_bos(model)
+ : std::rand() % n_vocab;
+ for (int i = 1; i < n_tokens; i++) {
+ tokens[i] = std::rand() % n_vocab;
+ }
+ llama_decode(
+ ctx, llama_batch_get_one(tokens.data(), n_tokens, n_processed, 0));
+ n_processed += n_tokens;
+ t_processed = n_processed;
+ }
+
+ llama_synchronize(ctx);
+
+ prompt_intervals.back().end = utils::get_time_ns();
+}
+
+void test::test_gen() {
+ llama_set_n_threads(ctx, n_threads, n_threads);
+
+ const llama_model *model = llama_get_model(ctx);
+ const int32_t n_vocab = llama_n_vocab(model);
+
+ llama_token token = llama_add_bos_token(model) ? llama_token_bos(model)
+ : std::rand() % n_vocab;
+
+ time_interval interval;
+ interval.start = utils::get_time_ns();
+ interval.end = 0;
+ gen_intervals.push_back(interval);
+
+ for (int i = 0; i < n_gen; i++) {
+ llama_decode(ctx, llama_batch_get_one(&token, 1, n_prompt + i, 0));
+ llama_synchronize(ctx);
+ if (i == 0) {
+ uint64_t ttft = utils::get_time_ns() - test_intervals.back().start;
+ time_to_first_token.push_back(ttft);
+ }
+ token = std::rand() % n_vocab;
+ t_gen = i + 1;
+ }
+
+ gen_intervals.back().end = utils::get_time_ns();
+}
+
+std::vector<uint64_t> test::get_samples_ns(token_metric metric) const {
+ const std::vector<time_interval> &intervals =
+ metric == PROMPT_TPS ? prompt_intervals
+ : metric == GEN_TPS ? gen_intervals
+ : test_intervals;
+
+ std::vector<uint64_t> samples_ns;
+ for (const auto &interval : intervals) {
+ if (interval.end == 0) {
+ continue;
+ }
+ samples_ns.push_back(interval.end - interval.start);
+ }
+ return samples_ns;
+}
+
+uint64_t test::avg_ns(token_metric metric) const {
+ std::vector<uint64_t> samples_ns = get_samples_ns(metric);
+ return utils::avg(samples_ns);
+}
+
+uint64_t test::stdev_ns(token_metric metric) const {
+ std::vector<uint64_t> samples_ns = get_samples_ns(metric);
+ return utils::stdev(samples_ns);
+}
+
+float test::get_power() const {
+ if (monitor_result.power > 0) {
+ return monitor_result.power;
+ } else {
+ // the sample is in mw, convert to w
+ return pwr_sampler->getLatestSample().power / 1000.0f;
+ }
+}
+
+std::vector<double> test::get_ts(token_metric metric) const {
+ int n_tokens = 0;
+ switch (metric) {
+ case TOTAL_TPS:
+ n_tokens = n_prompt + n_gen;
+ break;
+ case PROMPT_TPS:
+ n_tokens = n_prompt;
+ break;
+ case GEN_TPS:
+ n_tokens = n_gen;
+ break;
+ }
+
+ std::vector<double> ts;
+ std::vector<uint64_t> samples_ns = get_samples_ns(metric);
+ std::transform(samples_ns.begin(), samples_ns.end(), std::back_inserter(ts),
+ [n_tokens](uint64_t t) { return 1e9 * n_tokens / t; });
+ return ts;
+}
+
+double test::avg_ts(token_metric metric) const {
+ return utils::avg(get_ts(metric));
+}
+
+double test::stdev_ts(token_metric metric) const {
+ return utils::stdev(get_ts(metric));
+}
+
+double test::get_tps_watt(token_metric metric) const {
+ double power = get_power();
+ double ts = avg_ts(metric);
+
+ if (ts == 0.0 || power == 0.0) {
+ return 0.0;
+ }
+
+ return avg_ts(metric) / get_power();
+}
+
+double test::ttft() const {
+ if (time_to_first_token.empty()) {
+ return 0.0;
+ }
+ return utils::avg(time_to_first_token);
+}
+
+std::vector<std::string> test::get_values() const {
+ std::string tensor_split_str;
+ int max_nonzero = 0;
+ for (size_t i = 0; i < llama_max_devices(); i++) {
+ if (tensor_split[i] > 0) {
+ max_nonzero = i;
+ }
+ }
+ for (int i = 0; i <= max_nonzero; i++) {
+ char buf[32];
+ snprintf(buf, sizeof(buf), "%.2f", tensor_split[i]);
+ tensor_split_str += buf;
+ if (i < max_nonzero) {
+ tensor_split_str += "/";
+ }
+ }
+ double power = get_power();
+
+ std::vector<std::string> values = {
+ build_commit, std::to_string(build_number), model_name, model_quant_str,
+ model_params_str,
+ // std::to_string(cuda), std::to_string(opencl), std::to_string(vulkan),
+ // std::to_string(vulkan), std::to_string(metal), std::to_string(sycl),
+ // std::to_string(gpu_blas), std::to_string(blas), cpu_info, gpu_info,
+ model_filename, model_type, std::to_string(model_size),
+ std::to_string(model_n_params),
+ // std::to_string(n_batch), std::to_string(n_ubatch),
+ // std::to_string(n_threads), ggml_type_name(type_k),
+ // ggml_type_name(type_v), std::to_string(n_gpu_layers),
+ // split_mode_str(split_mode), std::to_string(main_gpu),
+ // std::to_string(no_kv_offload), std::to_string(flash_attn),
+ // tensor_split_str, std::to_string(use_mmap),
+ // std::to_string(embeddings),
+ std::to_string(n_prompt), std::to_string(n_gen), test_time,
+ std::to_string(avg_ns() / 1e6), std::to_string(stdev_ns() / 1e6),
+ std::to_string(avg_ts(PROMPT_TPS)),
+ std::to_string(get_tps_watt(PROMPT_TPS)),
+ std::to_string(stdev_ts(PROMPT_TPS)), std::to_string(avg_ts(GEN_TPS)),
+ std::to_string(get_tps_watt(GEN_TPS)),
+ std::to_string(stdev_ts(GEN_TPS)),
+ // name, std::to_string(power), std::to_string(monitor_result.vram),
+ // std::to_string(ttft() / 1e6)
+ name, std::to_string(power), std::to_string(ttft() / 1e6),
+ std::to_string(main_gpu)};
+ return values;
+}
+
+std::map<std::string, std::string> test::get_map() const {
+ std::map<std::string, std::string> map;
+ auto fields = get_fields();
+ auto values = get_values();
+ std::transform(fields.begin(), fields.end(), values.begin(),
+ std::inserter(map, map.end()),
+ std::make_pair<const std::string &, const std::string &>);
+ return map;
+}
+
+const std::vector<std::string> test::get_fields() {
+ static const std::vector<std::string> fields = {
+ "build_commit", "build_number", "model_name", "model_quant_str",
+ "model_params_str",
+ // "cuda", "opencl", "vulkan", "kompute", "metal", "sycl", "gpu_blas",
+ // "blas", "cpu_info", "gpu_info",
+ "model_filename", "model_type", "model_size", "model_n_params",
+ // "n_batch", "n_ubatch",
+ // "n_threads", "type_k", "type_v",
+ // "n_gpu_layers", "split_mode",
+ // "main_gpu", "no_kv_offload", "flash_attn",
+ // "tensor_split", "use_mmap", "embeddings",
+ "n_prompt", "n_gen", "test_time", "avg_time_ms", "stddev_time_ms",
+ "prompt_tps", "prompt_tps_watt", "prompt_tps_stddev", "gen_tps",
+ "gen_tps_watt", "gen_tps_stddev",
+ // "name", "power_watts", "vram_used_mb", "ttft_ms"
+ "name", "power_watts", "ttft_ms", "main_gpu"};
+ return fields;
+}
+
+std::string test::get_backend() {
+ if (cuda) {
+ return GGML_CUDA_NAME;
+ }
+ if (opencl) {
+ return "OpenCL";
+ }
+ if (vulkan) {
+ return "Vulkan";
+ }
+ if (kompute) {
+ return "Kompute";
+ }
+ if (metal) {
+ return "Metal";
+ }
+ if (gpu_blas) {
+ return "GPU BLAS";
+ }
+ if (blas) {
+ return "BLAS";
+ }
+
+ return "CPU";
+}
+
+test::field_type test::get_field_type(const std::string &field) {
+ if (field == "build_number" || field == "n_batch" || field == "n_ubatch" ||
+ field == "n_threads" || field == "model_size" ||
+ field == "model_n_params" || field == "n_gpu_layers" ||
+ field == "main_gpu" || field == "n_prompt" || field == "n_gen" ||
+ field == "avg_time_ms" || field == "stddev_time_ms" ||
+ field == "ttft_ms") {
+ return INT;
+ }
+ if (field == "cuda" || field == "opencl" || field == "vulkan" ||
+ field == "kompute" || field == "metal" || field == "gpu_blas" ||
+ field == "blas" || field == "sycl" || field == "f16_kv" ||
+ field == "no_kv_offload" || field == "flash_attn" ||
+ field == "use_mmap" || field == "embeddings") {
+ return BOOL;
+ }
+ if (field == "prompt_tps" || field == "prompt_tps_watt" ||
+ field == "prompt_tps_stddev" || field == "gen_tps" ||
+ field == "gen_tps_watt" || field == "gen_tps_stddev" ||
+ field == "power_watts" || field == "vram_used_mb") {
+ return FLOAT;
+ }
+ return STRING;
+}
\ No newline at end of file
diff --git a/localscore/benchmark.h b/localscore/benchmark.h
new file mode 100644
index 00000000..c69d3f52
--- /dev/null
+++ b/localscore/benchmark.h
@@ -0,0 +1,100 @@
+#pragma once
+
+#include "llama.cpp/llama.h"
+#include <atomic>
+#include <cstdint>
+#include <map>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "cmd.h"
+#include "powersampler.h"
+
+struct time_interval {
+ uint64_t start;
+ uint64_t end;
+};
+
+struct test_config {
+ int n_prompt;
+ int n_gen;
+};
+
+enum token_metric { TOTAL_TPS, PROMPT_TPS, GEN_TPS };
+
+struct test {
+ static const std::string build_commit;
+ static const int build_number;
+ static const bool cuda;
+ static const bool opencl;
+ static const bool vulkan;
+ static const bool kompute;
+ static const bool metal;
+ static const bool sycl;
+ static const bool gpu_blas;
+ static const bool blas;
+ static const std::string cpu_info;
+ static const std::string gpu_info;
+ std::string name;
+ std::string model_name;
+ std::string model_filename;
+ std::string model_type;
+ std::string model_quant_str;
+ std::string model_params_str;
+ uint64_t model_size;
+ uint64_t model_n_params;
+ int n_batch;
+ int n_ubatch;
+ int n_threads;
+ ggml_type type_k;
+ ggml_type type_v;
+ int n_gpu_layers;
+ llama_split_mode split_mode;
+ int main_gpu;
+ bool no_kv_offload;
+ bool flash_attn;
+ std::vector<float> tensor_split;
+ bool use_mmap;
+ bool embeddings;
+ int n_prompt;
+ int n_gen;
+ int reps;
+ mutable std::mutex t_gen_mutex;
+ std::atomic_bool test_completed{false};
+ volatile int curr_run;
+ volatile int t_gen; // this is the total number of tokens generated
+ volatile int t_processed; // this is the total number of tokens processed
+ power_sample_t monitor_result;
+ std::string test_time;
+ std::vector<time_interval> test_intervals;
+ std::vector<time_interval> prompt_intervals;
+ std::vector<time_interval> gen_intervals;
+ std::vector<uint64_t> time_to_first_token;
+ llama_context *ctx;
+ PowerSampler *pwr_sampler;
+
+ enum field_type { STRING, BOOL, INT, FLOAT };
+
+ test(const cmd_params &inst, const llama_model *lmodel,
+ llama_context *context, int repetitions, PowerSampler *sampler);
+ void run();
+ void test_prompt();
+ void test_gen();
+ std::vector<uint64_t> get_samples_ns(token_metric metric = TOTAL_TPS) const;
+ uint64_t avg_ns(token_metric metric = TOTAL_TPS) const;
+ uint64_t stdev_ns(token_metric metric = TOTAL_TPS) const;
+ float get_power() const;
+ std::vector<double> get_ts(token_metric metric = TOTAL_TPS) const;
+ double avg_ts(token_metric metric = TOTAL_TPS) const;
+ double stdev_ts(token_metric metric = TOTAL_TPS) const;
+ double get_tps_watt(token_metric metric = TOTAL_TPS) const;
+ double ttft() const;
+
+ std::vector<std::string> get_values() const;
+ std::map<std::string, std::string> get_map() const;
+
+ static std::string get_backend();
+ static const std::vector<std::string> get_fields();
+ static field_type get_field_type(const std::string &field);
+};
diff --git a/localscore/cmd.cpp b/localscore/cmd.cpp
new file mode 100644
index 00000000..51cc95a2
--- /dev/null
+++ b/localscore/cmd.cpp
@@ -0,0 +1,201 @@
+#include <algorithm>
+
+#include "cmd.h"
+#include "llama.cpp/cores.h"
+#include "system.h"
+#include <cosmo.h>
+
+static const cmd_params cmd_params_defaults = {
+ /* model */ "", // [jart] no default guessing
+ /* n_prompt */ 0,
+ /* n_gen */ 0,
+ /* n_batch */ 2048,
+ /* n_ubatch */ 512,
+ /* type_k */ X86_HAVE(AVX512_BF16) ? GGML_TYPE_BF16 : GGML_TYPE_F16,
+ /* type_v */ X86_HAVE(AVX512_BF16) ? GGML_TYPE_BF16 : GGML_TYPE_F16,
+ /* n_threads */ cpu_get_num_math(),
+ /* gpu */ LLAMAFILE_GPU_AUTO,
+ /* n_gpu_layers */ 9999,
+ /* split_mode */ LLAMA_SPLIT_MODE_NONE,
+ /* main_gpu */ UINT_MAX,
+ /* no_kv_offload */ false,
+ /* flash_attn */ false,
+ /* tensor_split */ std::vector<float>(llama_max_devices(), 0.0f),
+ /* use_mmap */ true,
+ /* embeddings */ false,
+ /* numa */ GGML_NUMA_STRATEGY_DISABLED,
+ /* reps */ 1,
+ /* verbose */ false,
+ /* send_results */ SEND_ASK,
+ /* output_format */ CONSOLE,
+};
+
+llama_model_params cmd_params::to_llama_mparams() const {
+ llama_model_params mparams = llama_model_default_params();
+
+ mparams.n_gpu_layers = n_gpu_layers;
+ mparams.split_mode = split_mode;
+ mparams.main_gpu = main_gpu;
+ mparams.tensor_split = tensor_split.data();
+ mparams.use_mmap = use_mmap;
+
+ return mparams;
+}
+
+bool cmd_params::equal_mparams(const cmd_params & other) const {
+ return model == other.model &&
+ n_gpu_layers == other.n_gpu_layers &&
+ split_mode == other.split_mode &&
+ main_gpu == other.main_gpu &&
+ use_mmap == other.use_mmap &&
+ tensor_split == other.tensor_split;
+}
+
+llama_context_params cmd_params::to_llama_cparams() const {
+ llama_context_params cparams = llama_context_default_params();
+
+ cparams.n_ctx = n_prompt + n_gen;
+ cparams.n_batch = n_batch;
+ cparams.n_ubatch = n_ubatch;
+ cparams.type_k = type_k;
+ cparams.type_v = type_v;
+ cparams.offload_kqv = !no_kv_offload;
+ cparams.flash_attn = flash_attn;
+ cparams.embeddings = embeddings;
+
+ return cparams;
+}
+
+cmd_params parse_cmd_params(int argc, char ** argv) {
+ cmd_params params = cmd_params_defaults;
+ std::string arg;
+ bool invalid_param = false;
+ const std::string arg_prefix = "--";
+
+ for (int i = 1; i < argc; i++) {
+ arg = argv[i];
+ if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {
+ std::replace(arg.begin(), arg.end(), '_', '-');
+ }
+
+ if (arg == "-h" || arg == "--help") {
+ print_usage(argc, argv);
+ exit(0);
+ } else if (arg == "-m" || arg == "--model") {
+ if (++i >= argc) {
+ invalid_param = true;
+ }
+ else params.model = argv[i];
+ } else if (arg == "-i" || arg == "--gpu-index") {
+ if (++i >= argc) {
+ invalid_param = true;
+ }
+ else params.main_gpu = std::stoi(argv[i]);
+ } else if (arg == "--list-gpus") {
+ FLAG_gpu = LLAMAFILE_GPU_AUTO;
+ FLAGS_READY = true;
+ list_available_accelerators(); // You'll need to implement this
+ exit(0);
+ } else if (arg == "-c" || arg == "--cpu") {
+ FLAG_gpu = LLAMAFILE_GPU_DISABLE;
+ params.n_gpu_layers = 0;
+ } else if (arg == "-g" || arg == "--gpu") {
+ if (++i >= argc) {
+ invalid_param = true;
+ }
+ else {
+ FLAG_gpu = llamafile_gpu_parse(argv[i]);
+ if (FLAG_gpu == LLAMAFILE_GPU_ERROR) {
+ fprintf(stderr, "error: invalid --gpu value: %s\n", argv[i]);
+ exit(1);
+ }
+ params.n_gpu_layers = (FLAG_gpu >= 0) ? 9999 : 0;
+ }
+ } else if (arg == "-o" || arg == "--output") {
+ if (++i >= argc) {
+ invalid_param = true;
+ }
+ else {
+ std::string fmt = argv[i];
+ if (fmt == "csv") params.output_format = CSV;
+ else if (fmt == "json") params.output_format = JSON;
+ else if (fmt == "console") params.output_format = CONSOLE;
+ else {
+ invalid_param = true;
+ }
+ }
+ } else if (arg == "-v" || arg == "--verbose") {
+ params.verbose = true;
+ } else if (arg == "-y" || arg == "--send-results") {
+ params.send_results = SEND_YES;
+ } else if (arg == "-n" || arg == "--no-send-results") {
+ params.send_results = SEND_NO;
+ } else if (arg == "--extended") {
+ params.reps = 4;
+ } else if (arg == "--long") {
+ params.reps = 16;
+ } else if (arg == "--reps") {
+ if (++i >= argc) {
+ invalid_param = true;
+ }
+ else params.reps = std::max(1, std::stoi(argv[i]));
+ } else if (arg == "--recompile") {
+ FLAG_recompile = true;
+ } else if (arg == "--localscore") {
+ } else if (arg[0] == '-') {
+ invalid_param = true;
+ } else {
+ params.model = argv[i];
+ }
+
+ if (invalid_param) break;
+ }
+
+ if (invalid_param) {
+ fprintf(stderr, "%s: invalid parameter for: %s\n",
+ program_invocation_name, arg.c_str());
+ exit(1);
+ }
+
+ if (params.model.empty()) {
+ fprintf(stderr, "%s: missing model file\n", program_invocation_name);
+ exit(1);
+ }
+
+ // Validate mutually exclusive flags
+ if (params.send_results == SEND_YES && params.send_results == SEND_NO) {
+ fprintf(stderr, "%s: cannot use both --send-results and --no-send-results\n",
+ program_invocation_name);
+ exit(1);
+ }
+
+ return params;
+}
+
+static const char * output_format_str(output_formats format) {
+ switch (format) {
+ case CSV: return "csv";
+ case JSON: return "json";
+ case CONSOLE: return "console";
+ default: GGML_ASSERT(!"invalid output format");
+ }
+}
+
+void print_usage(int /* argc */, char ** argv) {
+ printf("usage: %s [options]\n", argv[0]);
+ printf("\n");
+ printf("options:\n");
+ printf(" -h, --help\n");
+ printf(" -m, --model <filename> (default: %s)\n", cmd_params_defaults.model.c_str());
+ printf(" -c, --cpu disable GPU acceleration (alias for --gpu=disabled)\n");
+ printf(" -g, --gpu <auto|amd|apple|nvidia|disabled> (default: \"auto\")\n");
+ printf(" -i, --gpu-index <i> select GPU by index (default: %d)\n", cmd_params_defaults.main_gpu);
+ printf(" --list-gpus list available GPUs and exit\n");
+ printf(" -o, --output <csv|json|console> (default: %s)\n", output_format_str(cmd_params_defaults.output_format));
+ printf(" -v, --verbose verbose output (default: %s)\n", cmd_params_defaults.verbose ? "yes" : "no");
+ printf(" -y, --send-results send results without confirmation\n");
+ printf(" -n, --no-send-results disable sending results\n");
+ printf(" -e, --extended run 4 reps (shortcut for --reps=4)\n");
+ printf(" --long run 16 reps (shortcut for --reps=16)\n");
+ printf(" --reps <N> set custom number of repetitions\n");
+}
\ No newline at end of file
diff --git a/localscore/cmd.h b/localscore/cmd.h
new file mode 100644
index 00000000..413aa705
--- /dev/null
+++ b/localscore/cmd.h
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include "llama.cpp/llama.h"
+
+enum output_formats {CSV, JSON, CONSOLE};
+enum send_results_mode {SEND_ASK, SEND_YES, SEND_NO};
+
+struct cmd_params {
+ std::string model;
+ int n_prompt;
+ int n_gen;
+ int n_batch;
+ int n_ubatch;
+ ggml_type type_k;
+ ggml_type type_v;
+ int n_threads;
+ int gpu;
+ int n_gpu_layers;
+ llama_split_mode split_mode;
+ unsigned int main_gpu;
+ bool no_kv_offload;
+ bool flash_attn;
+ std::vector<float> tensor_split;
+ bool use_mmap;
+ bool embeddings;
+ ggml_numa_strategy numa;
+ int reps;
+ bool verbose;
+ send_results_mode send_results;
+ output_formats output_format;
+
+ llama_model_params to_llama_mparams() const;
+ bool equal_mparams(const cmd_params & other) const;
+ llama_context_params to_llama_cparams() const;
+};
+
+cmd_params parse_cmd_params(int argc, char ** argv);
+void print_usage(int argc, char ** argv);
\ No newline at end of file
diff --git a/localscore/cuda.bat b/localscore/cuda.bat
new file mode 100644
index 00000000..f269b00e
--- /dev/null
+++ b/localscore/cuda.bat
@@ -0,0 +1,50 @@
+:: Compiles distributable DLL for NVIDIA GPU support
+::
+:: The artifact will only depend on KERNEL32.DLL and NVCUDA.DLL.
+:: NVCUDA DLLs are provided by the installation of the windows GPU
+:: driver on a Windows system that has a CUDA-capable GPU installed.
+
+mkdir build\release 2>nul
+
+for %%f in (
+ llama.cpp\ggml-cuda.cu
+ llama.cpp\ggml-cuda.h
+ llama.cpp\ggml-impl.h
+ llama.cpp\ggml-alloc.h
+ llama.cpp\ggml-common.h
+ llama.cpp\ggml-backend.h
+ llama.cpp\ggml-backend-impl.h
+ llama.cpp\ggml.h
+ llamafile\tinyblas.h
+ llamafile\tinyblas.cu
+ llamafile\llamafile.h
+) do copy %%f build\release
+
+cd build\release
+
+nvcc --shared ^
+ --use_fast_math ^
+ -gencode arch=compute_60,code=sm_60 ^
+ -gencode arch=compute_61,code=sm_61 ^
+ -gencode arch=compute_70,code=sm_70 ^
+ -gencode arch=compute_75,code=sm_75 ^
+ -gencode arch=compute_80,code=sm_80 ^
+ -gencode arch=compute_86,code=sm_86 ^
+ -gencode arch=compute_89,code=sm_89 ^
+ -gencode arch=compute_90,code=sm_90 ^
+ -gencode arch=compute_120,code=sm_120 ^
+ --forward-unknown-to-host-compiler ^
+ -Xcompiler="/nologo /EHsc /O2 /GR /MT" ^
+ -DNDEBUG ^
+ -DGGML_BUILD=1 ^
+ -DGGML_SHARED=1 ^
+ -DGGML_CUDA_MMV_Y=1 ^
+ -DGGML_MULTIPLATFORM ^
+ -DGGML_CUDA_DMMV_X=32 ^
+ -DK_QUANTS_PER_ITERATION=2 ^
+ -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 ^
+ -DGGML_MINIMIZE_CODE_SIZE ^
+ -DGGML_USE_CUBLAS ^
+ -o ggml-cuda.dll ^
+ ggml-cuda.cu ^
+ -lcuda -lcublas
diff --git a/localscore/cuda.sh b/localscore/cuda.sh
new file mode 100755
index 00000000..528651e0
--- /dev/null
+++ b/localscore/cuda.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+TMP=$(mktemp -d) || exit
+
+cp llama.cpp/ggml-cuda.cu \
+ llama.cpp/ggml-cuda.h \
+ llama.cpp/ggml-impl.h \
+ llama.cpp/ggml-alloc.h \
+ llama.cpp/ggml-common.h \
+ llama.cpp/ggml-backend.h \
+ llama.cpp/ggml-backend-impl.h \
+ llama.cpp/ggml.h \
+ llamafile/tinyblas.h \
+ llamafile/tinyblas.cu \
+ llamafile/llamafile.h \
+ llamafile/rocm.bat \
+ llamafile/rocm.sh \
+ llamafile/cuda.bat \
+ llamafile/cuda.sh \
+ "$TMP" || exit
+
+cd "$TMP"
+
+/usr/local/cuda/bin/nvcc \
+ --shared \
+ --use_fast_math \
+ -gencode arch=compute_60,code=sm_60 \
+ -gencode arch=compute_61,code=sm_61 \
+ -gencode arch=compute_70,code=sm_70 \
+ -gencode arch=compute_75,code=sm_75 \
+ -gencode arch=compute_80,code=sm_80 \
+ -gencode arch=compute_86,code=sm_86 \
+ -gencode arch=compute_89,code=sm_89 \
+ -gencode arch=compute_90,code=sm_90 \
+ --forward-unknown-to-host-compiler \
+ --compiler-options "-fPIC -O2" \
+ -DNDEBUG \
+ -DGGML_BUILD=1 \
+ -DGGML_SHARED=1 \
+ -DGGML_CUDA_MMV_Y=1 \
+ -DGGML_MULTIPLATFORM \
+ -DGGML_CUDA_DMMV_X=32 \
+ -DK_QUANTS_PER_ITERATION=2 \
+ -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 \
+ -DGGML_MINIMIZE_CODE_SIZE \
+ -DGGML_USE_CUBLAS \
+ -o ~/ggml-cuda.localscore.so \
+ ggml-cuda.cu \
+ -lcublas \
+ -lcuda
diff --git a/localscore/doc/troubleshooting.md b/localscore/doc/troubleshooting.md
new file mode 100644
index 00000000..290b8537
--- /dev/null
+++ b/localscore/doc/troubleshooting.md
@@ -0,0 +1,98 @@
+# Troubleshooting the LocalScore CLI
+
+Much of the troubleshooting for LocalScore is similar to Llamafile.
+
+## CLI
+
+### Windows
+
+On Windows, there's a variety of issues.
+
+#### Model Size
+
+If you are using a Llamafile or LocalScore benchmark bundle larger than 4 gigabytes, you will not be able to run it on Windows due to Windows limitations. You can use LocalScore as a standalone utility and pass in models in GGUF format.
+
+#### WSL2
+
+On WSL, there are many possible gotchas. One thing that helps solve them
+completely is this:
+
+```
+[Unit]
+Description=cosmopolitan APE binfmt service
+After=wsl-binfmt.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register"
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Put that in `/etc/systemd/system/cosmo-binfmt.service`.
+
+Ensure that the APE loader is installed to `/usr/bin/ape`:
+
+```sh
+sudo wget -O /usr/bin/ape https://cosmo.zip/pub/cosmos/bin/ape-$(uname -m).elf
+sudo chmod +x /usr/bin/ape
+```
+
+Then run `sudo systemctl enable --now cosmo-binfmt`.
+
+Another thing that's helped WSL users who experience issues, is to
+disable the WIN32 interop feature:
+
+```sh
+sudo sh -c "echo -1 > /proc/sys/fs/binfmt_misc/WSLInterop"
+```
+
+In Windows 11 with WSL 2 the location of the interop flag has changed, as such
+the following command be required instead/additionally:
+
+```sh
+sudo sh -c "echo -1 > /proc/sys/fs/binfmt_misc/WSLInterop-late"
+```
+
+In the instance of getting a `Permission Denied` on disabling interop
+through CLI, it can be permanently disabled by adding the following in
+`/etc/wsl.conf`
+
+```sh
+[interop]
+enabled=false
+```
+
+#### Other Issues
+
+We have observed that on Windows the performance of LocalScore is slower than on Linux. This is expected at the moment.
+
+### Linux
+
+On some Linux systems, you might get errors relating to `run-detectors`
+or WINE. This is due to `binfmt_misc` registrations. You can fix that by
+adding an additional registration for the APE file format llamafile
+uses:
+
+```sh
+sudo wget -O /usr/bin/ape https://cosmo.zip/pub/cosmos/bin/ape-$(uname -m).elf
+sudo chmod +x /usr/bin/ape
+sudo sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register"
+sudo sh -c "echo ':APE-jart:M::jartsr::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register"
+```
+
+### Mac
+
+On macOS with Apple Silicon you need to have Xcode Command Line Tools
+installed for llamafile to be able to bootstrap itself.
+
+If you use zsh and have trouble running llamafile, try saying `sh -c
+./llamafile`. This is due to a bug that was fixed in zsh 5.9+. The same
+is the case for Python `subprocess`, old versions of Fish, etc.
+
+
+#### Mac error "... cannot be opened because the developer cannot be verified"
+
+1. Immediately launch System Settings, then go to Privacy & Security. llamafile should be listed at the bottom, with a button to Allow.
+2. If not, then change your command in the Terminal to be `sudo spctl --master-disable; [llama launch command]; sudo spctl --master-enable`. This is because `--master-disable` disables _all_ checking, so you need to turn it back on after quitting llama.
\ No newline at end of file
diff --git a/localscore/http.cpp b/localscore/http.cpp
new file mode 100644
index 00000000..fbff82ad
--- /dev/null
+++ b/localscore/http.cpp
@@ -0,0 +1,502 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "llamafile/macros.h"
+#include "llamafile/net.h"
+#include "llamafile/string.h"
+#include "third_party/mbedtls/ctr_drbg.h"
+#include "third_party/mbedtls/debug.h"
+#include "third_party/mbedtls/error.h"
+#include "third_party/mbedtls/iana.h"
+#include "third_party/mbedtls/net_sockets.h"
+#include "third_party/mbedtls/ssl.h"
+#include "third_party/mbedtls/x509.h"
+#include <cosmo.h>
+#include <errno.h>
+#include <net/http/http.h>
+#include <net/http/url.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <string>
+#include <vector>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#include <format>
+
+#include "http.h"
+
+/**
+ * @fileoverview Downloads HTTP URL to stdout.
+ */
+
+static const char *prog;
+
+static wontreturn void PrintUsage(int fd, int rc) {
+ tinyprint(fd, "usage: ", prog, " [-iksvV] URL\n", NULL);
+ exit(rc);
+}
+
+static const char *DescribeErrno(void) {
+ return strerror(errno);
+}
+
+static int GetSslEntropy(void *c, unsigned char *p, size_t n) {
+ if (getrandom(p, n, 0) != n) {
+ perror("getrandom");
+ exit(1);
+ }
+ return 0;
+}
+
+static void OnSslDebug(void *ctx, int level, const char *file, int line, const char *message) {
+ char sline[12];
+ char slevel[12];
+ FormatInt32(sline, line);
+ FormatInt32(slevel, level);
+ tinyprint(2, file, ":", sline, ": (", slevel, ") ", message, "\n", NULL);
+}
+
+static int TlsSend(void *c, const unsigned char *p, size_t n) {
+ int rc;
+ if ((rc = write(*(int *)c, p, n)) == -1) {
+ perror("TlsSend");
+ exit(1);
+ }
+ return rc;
+}
+
+static int TlsRecv(void *c, unsigned char *p, size_t n, uint32_t o) {
+ int r;
+ struct iovec v[2];
+ static unsigned a, b;
+ static unsigned char t[4096];
+ if (a < b) {
+ r = MIN(n, b - a);
+ memcpy(p, t + a, r);
+ if ((a += r) == b) {
+ a = b = 0;
+ }
+ return r;
+ }
+ v[0].iov_base = p;
+ v[0].iov_len = n;
+ v[1].iov_base = t;
+ v[1].iov_len = sizeof(t);
+ if ((r = readv(*(int *)c, v, 2)) == -1) {
+ perror("TlsRecv");
+ exit(1);
+ }
+ if (r > n) {
+ b = r - n;
+ }
+ return MIN(n, r);
+}
+
+struct ParsedUrl {
+ std::string host;
+ std::string port;
+ std::string path;
+};
+
+struct SSLContext {
+ mbedtls_ssl_context ssl;
+ mbedtls_ssl_config conf;
+ mbedtls_ctr_drbg_context drbg;
+
+ SSLContext() {
+ mbedtls_ssl_init(&ssl);
+ mbedtls_ssl_config_init(&conf);
+ mbedtls_ctr_drbg_init(&drbg);
+ }
+
+ ~SSLContext() {
+ mbedtls_ssl_free(&ssl);
+ mbedtls_ssl_config_free(&conf);
+ mbedtls_ctr_drbg_free(&drbg);
+ }
+};
+
+static ParsedUrl ExtractUrlComponents(const std::string& url_str, bool* usessl) {
+ ParsedUrl result;
+ struct Url url;
+
+ /*
+ * Parse URL.
+ */
+ gc(ParseUrl(url_str.c_str(), -1, &url, kUrlPlus));
+ gc(url.params.p);
+
+ // Check scheme
+ if (url.scheme.n) {
+ if (url.scheme.n == 5 && !memcasecmp(url.scheme.p, "https", 5)) {
+ *usessl = true;
+ } else if (!(url.scheme.n == 4 && !memcasecmp(url.scheme.p, "http", 4))) {
+ printf("not an http/https url: %s\n", url_str.c_str());
+ exit(1);
+ }
+ }
+
+ // Set host and port
+ if (url.host.n) {
+ // Copy the host data into the string
+ result.host = std::string(url.host.p, url.host.n);
+ if (url.port.n) {
+ result.port = std::string(url.port.p, url.port.n);
+ } else {
+ result.port = *usessl ? "443" : "80";
+ }
+ } else {
+ result.host = "127.0.0.1";
+ result.port = *usessl ? "443" : "80";
+ }
+
+ // Validate host
+ if (!IsAcceptableHost(result.host.c_str(), -1)) {
+ printf("invalid host: %s\n", url_str.c_str());
+ exit(1);
+ }
+
+ // Handle path
+ if (!url.path.n || url.path.p[0] != '/') {
+ // If path is empty or doesn't start with '/', prepend '/'
+ result.path = "/" + std::string(url.path.p, url.path.n);
+ } else {
+ result.path = std::string(url.path.p, url.path.n);
+ }
+
+ return result;
+}
+
+static std::string BuildHTTPRequest(const ParsedUrl url, const Headers& headers, const std::string& body = "") {
+ std::string request;
+ request += std::format("POST {} HTTP/1.1\r\n"
+ "Host: {}\r\n"
+ "Connection: close\r\n",
+ url.path, url.host);
+
+ // write all the headers. iterate through the map and write them to request
+ for (auto const& [key, val] : headers) {
+ request += key;
+ request += ": ";
+ request += val;
+ request += "\r\n";
+ }
+
+ if (!body.empty()) {
+ // Add Content-Length header
+ char length_str[21];
+ FormatUint64(length_str, body.size());
+ request += "Content-Length: ";
+ request += length_str;
+ request += "\r\n";
+
+ // End headers and add payload
+ request += "\r\n";
+ request += body;
+ }
+
+ return request;
+}
+
+static int ConnectToServer(const ParsedUrl url) {
+ int sock = -1;
+ struct addrinfo *addr = nullptr;
+ struct addrinfo hints = {};
+
+ // Initialize hints structure
+ hints.ai_family = AF_UNSPEC;
+ hints.ai_socktype = SOCK_STREAM;
+ hints.ai_protocol = IPPROTO_TCP;
+ hints.ai_flags = AI_NUMERICSERV;
+
+ // Perform DNS lookup
+ int gai_result = getaddrinfo(url.host.c_str(), url.port.c_str(), &hints, &addr);
+ if (gai_result != 0) {
+ tinyprint(2, prog, ": could not resolve host: ", url.host.c_str(), "\n", NULL);
+ return -1;
+ }
+
+ // cleanup using a unique_ptr with custom deleter
+ std::unique_ptr<struct addrinfo, decltype(&freeaddrinfo)> addr_guard(addr, freeaddrinfo);
+
+ // Connect to server
+ struct timeval tt = {-60};
+ sock = lf::socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol, false, &tt);
+ if (sock == -1) {
+ perror("socket");
+ return -1;
+ }
+
+ // Try to connect
+ if (connect(sock, addr->ai_addr, addr->ai_addrlen)) {
+ tinyprint(2, prog, ": failed to connect to ", url.host.c_str(),
+ " port ", url.port.c_str(), ": ", DescribeErrno(), "\n", NULL);
+ close(sock); // Clean up the socket if connect fails
+ return -1;
+ }
+
+ return sock;
+}
+
+static std::unique_ptr<SSLContext> SetupSSL(int& sock, const std::string& hostname) {
+ auto ctx = std::make_unique<SSLContext>();
+
+ // Setup SSL configuration
+ unassert(!mbedtls_ctr_drbg_seed(&ctx->drbg, GetSslEntropy, 0, "justine", 7));
+ unassert(!mbedtls_ssl_config_defaults(&ctx->conf,
+ MBEDTLS_SSL_IS_CLIENT,
+ MBEDTLS_SSL_TRANSPORT_STREAM,
+ MBEDTLS_SSL_PRESET_SUITEC));
+
+ mbedtls_ssl_conf_authmode(&ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);
+ mbedtls_ssl_conf_ca_chain(&ctx->conf, lf::sslroots(), 0);
+ mbedtls_ssl_conf_rng(&ctx->conf, mbedtls_ctr_drbg_random, &ctx->drbg);
+ #ifndef NDEBUG
+ mbedtls_ssl_conf_dbg(&ctx->conf, OnSslDebug, 0);
+ #endif
+
+ // Setup SSL context
+ unassert(!mbedtls_ssl_setup(&ctx->ssl, &ctx->conf));
+ unassert(!mbedtls_ssl_set_hostname(&ctx->ssl, hostname.c_str()));
+ mbedtls_ssl_set_bio(&ctx->ssl, &sock, TlsSend, 0, TlsRecv);
+
+ // Perform handshake
+ int ret = mbedtls_ssl_handshake(&ctx->ssl);
+ if (ret != 0) {
+ throw std::runtime_error("SSL handshake failed: " +
+ std::string(DescribeSslClientHandshakeError(&ctx->ssl, ret)));
+ }
+
+ return ctx;
+}
+
+static void SendRequest(const std::string& request, int sock, SSLContext* ssl_ctx) {
+ size_t sent = 0;
+ while (sent < request.size()) {
+ ssize_t rc;
+ if (ssl_ctx) {
+ rc = mbedtls_ssl_write(&ssl_ctx->ssl,
+ reinterpret_cast<const unsigned char*>(request.data() + sent),
+ request.size() - sent);
+ } else {
+ rc = write(sock, request.data() + sent, request.size() - sent);
+ }
+
+ if (rc <= 0) {
+ throw std::runtime_error("Failed to send request");
+ }
+ sent += rc;
+ }
+}
+
+bool hasHeader(const HttpMessage& msg, int header) {
+ return msg.headers[header].a != 0;
+}
+
+const char* getHeaderData(const HttpMessage& msg, int header, const char* base) {
+ return base + msg.headers[header].a;
+}
+
+size_t getHeaderLength(const HttpMessage& msg, int header) {
+ return msg.headers[header].b - msg.headers[header].a;
+}
+
+bool headerEqualCase(const HttpMessage& msg, int header, const char* str, const char* base) {
+ return SlicesEqualCase(str, strlen(str),
+ getHeaderData(msg, header, base),
+ getHeaderLength(msg, header));
+}
+
+static Response DecodeHttpResponse(int sock, SSLContext* ssl_ctx, size_t initial_buffer_size = 1000) {
+ std::vector<char> buffer;
+ buffer.reserve(initial_buffer_size);
+
+ size_t current_pos = 0;
+ Response response;
+ struct HttpMessage msg;
+ struct HttpUnchunker chunker;
+
+ InitHttpMessage(&msg, kHttpResponse);
+
+ enum class ParseState {
+ Headers = 0,
+ Body,
+ BodyLengthed,
+ BodyChunked
+ } state = ParseState::Headers;
+
+ auto read_chunk = [sock, ssl_ctx](char* buf, size_t len) -> ssize_t {
+ if (ssl_ctx) {
+ return mbedtls_ssl_read(&ssl_ctx->ssl,
+ reinterpret_cast<unsigned char*>(buf),
+ len);
+ } else {
+ return read(sock, buf, len);
+ }
+ };
+
+ size_t header_length = 0;
+ size_t payload_length = 0;
+ bool headers_parsed = false;
+
+ while (true) {
+ if (current_pos == buffer.size()) {
+ buffer.resize(buffer.size() + initial_buffer_size);
+ }
+
+ ssize_t bytes_read = read_chunk(&buffer[current_pos],
+ buffer.size() - current_pos);
+
+ if (bytes_read < 0) {
+ if (ssl_ctx && bytes_read == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) {
+ break;
+ }
+ throw std::runtime_error("Failed to read response");
+ }
+
+ if (bytes_read == 0 && state != ParseState::Headers) {
+ break;
+ }
+
+ current_pos += bytes_read;
+
+ if (!headers_parsed) {
+ int rc = ParseHttpMessage(&msg, buffer.data(), current_pos, buffer.size());
+ if (rc == -1) {
+ throw std::runtime_error("Invalid HTTP message");
+ }
+ if (rc > 0) {
+ header_length = rc;
+ response.status = msg.status;
+ response.raw_headers = std::string(buffer.data(), header_length);
+ headers_parsed = true;
+
+ if (hasHeader(msg, kHttpTransferEncoding) &&
+ !headerEqualCase(msg, kHttpTransferEncoding, "identity", buffer.data())) {
+ if (!headerEqualCase(msg, kHttpTransferEncoding, "chunked", buffer.data())) {
+ throw std::runtime_error("Unsupported transfer encoding");
+ }
+ state = ParseState::BodyChunked;
+ memset(&chunker, 0, sizeof(chunker));
+ } else if (hasHeader(msg, kHttpContentLength)) {
+ int content_length = ParseContentLength(
+ getHeaderData(msg, kHttpContentLength, buffer.data()),
+ getHeaderLength(msg, kHttpContentLength));
+ if (content_length == -1) {
+ throw std::runtime_error("Invalid content length");
+ }
+ response.content_length = content_length;
+ state = ParseState::BodyLengthed;
+ } else {
+ state = ParseState::Body;
+ }
+
+ // Process any body data that came with the headers
+ if (current_pos > header_length) {
+ goto process_body;
+ }
+ continue;
+ }
+ } else {
+process_body:
+ switch (state) {
+ case ParseState::Body: {
+ response.body.append(buffer.data() + header_length,
+ current_pos - header_length);
+ break;
+ }
+
+ case ParseState::BodyLengthed: {
+ size_t remaining = response.content_length - response.body.size();
+ size_t to_copy = std::min(remaining,
+ current_pos - header_length - response.body.size());
+ response.body.append(buffer.data() + header_length + response.body.size(),
+ to_copy);
+ if (response.body.size() >= response.content_length) {
+ return response;
+ }
+ break;
+ }
+
+ case ParseState::BodyChunked: {
+ size_t chunk_length;
+ int rc = Unchunk(&chunker,
+ buffer.data() + header_length,
+ current_pos - header_length,
+ &chunk_length);
+ if (rc == -1) {
+ throw std::runtime_error("Invalid chunk encoding");
+ }
+ if (rc > 0) {
+ response.body.append(buffer.data() + header_length, chunk_length);
+ return response;
+ }
+ break;
+ }
+
+ default:
+ break;
+ }
+ }
+ }
+
+ return response;
+}
+
+Response SendHttpRequest(const std::string& url_str, uint64_t method,
+ const Headers& headers, const std::string& body = "") {
+ const char *agent = "hurl/1.o (https://github.com/jart/cosmopolitan)";
+ bool usessl = false;
+
+ ParsedUrl url = ExtractUrlComponents(url_str, &usessl);
+
+ std::string request = (method == kHttpGet)
+ ? BuildHTTPRequest(url, headers)
+ : BuildHTTPRequest(url, headers, body);
+
+ int sock = ConnectToServer(url);
+
+ std::unique_ptr<SSLContext> ssl_ctx;
+ if (usessl) {
+ try {
+ ssl_ctx = SetupSSL(sock, url.host);
+ } catch (const std::exception& e) {
+ printf("Error setting up SSL: %s\n", e.what());
+ close(sock);
+ return Response();
+ }
+ }
+
+ SendRequest(request, sock, ssl_ctx.get());
+ Response resp = DecodeHttpResponse(sock, ssl_ctx.get());
+ close(sock);
+ return resp;
+}
+
+Response GET(const std::string& url_str, const Headers& headers) {
+ return SendHttpRequest(url_str, kHttpGet, headers);
+}
+
+Response POST(const std::string& url_str, const std::string& body, const Headers& headers) {
+ return SendHttpRequest(url_str, kHttpPost, headers, body);
+}
\ No newline at end of file
diff --git a/localscore/http.h b/localscore/http.h
new file mode 100644
index 00000000..54a9e3a2
--- /dev/null
+++ b/localscore/http.h
@@ -0,0 +1,16 @@
+#include <string>
+#include <map>
+
+typedef std::map<std::string, std::string> Headers;
+
+struct Response {
+ std::string raw_headers;
+ std::string body;
+ int status;
+ size_t content_length;
+ bool is_chunked;
+};
+
+Response GET(const std::string& url, const Headers& headers = {});
+
+Response POST(const std::string& url, const std::string& body, const Headers& headers = {});
\ No newline at end of file
diff --git a/localscore/localscore-example.jpeg b/localscore/localscore-example.jpeg
new file mode 100644
index 00000000..3302f467
Binary files /dev/null and b/localscore/localscore-example.jpeg differ
diff --git a/localscore/localscore.cpp b/localscore/localscore.cpp
new file mode 100644
index 00000000..a72e28a6
--- /dev/null
+++ b/localscore/localscore.cpp
@@ -0,0 +1,538 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+
+#include <algorithm>
+#include <cassert>
+// #include <chrono> [jart]
+#include <clocale>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <ctime>
+#include <cstdlib>
+#include <map>
+#include <sstream>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <cosmo.h>
+#include <dlfcn.h>
+#include <libgen.h>
+#include <pthread.h>
+#include <sys/stat.h>
+#include <libc/intrin/x86.h>
+#include <libc/sysv/consts/hwcap.h>
+
+#include <sys/utsname.h>
+#include <sys/sysinfo.h>
+#include <unistd.h>
+
+#include "http.h"
+#include "powersampler.h"
+#include "ascii_digits.h"
+#include "system.h"
+#include "cmd.h"
+#include "benchmark.h"
+#include "printer.h"
+
+#include "localscore.h"
+
+#include "llama.cpp/cores.h"
+#include "llama.cpp/ggml.h"
+#include "llama.cpp/ggml-metal.h"
+#include "llama.cpp/llama.h"
+#include "llama.cpp/string.h"
+#include "llama.cpp/common.h"
+#include "llama.cpp/ggml-cuda.h"
+
+#include "llamafile/llamafile.h"
+#include "llamafile/compute.h"
+#include "llamafile/json.h"
+
+using jt::Json;
+
+const std::string BASE_LOCALSCORE_URL = "https://www.localscore.ai";
+
+const std::string test::build_commit = LLAMA_COMMIT;
+const int test::build_number = LLAMA_BUILD_NUMBER;
+const bool test::cuda = false; // !!ggml_cpu_has_cuda(); // [jart]
+const bool test::opencl = false; // !!ggml_cpu_has_clblast(); // [jart]
+const bool test::vulkan = false; // !!ggml_cpu_has_vulkan(); // [jart]
+const bool test::kompute = false; // !!ggml_cpu_has_kompute(); // [jart]
+const bool test::metal = false; // !!ggml_cpu_has_metal(); // [jart]
+const bool test::gpu_blas = false; // !!ggml_cpu_has_gpublas(); // [jart]
+const bool test::blas = false; // !!ggml_cpu_has_blas(); // [jart]
+const bool test::sycl = false; // !!ggml_cpu_has_sycl(); // [jart]
+const std::string test::cpu_info = llamafile_describe_cpu();
+const std::string test::gpu_info = ""; //get_gpu_info(); // [jart]
+
+static void llama_null_log_callback(enum ggml_log_level level, const char * text, void * user_data) {
+ (void) level;
+ (void) text;
+ (void) user_data;
+}
+
+struct update_t_gen_column_args {
+ const test & t;
+ printer* p;
+};
+
+void* update_t_gen_column(void* args) {
+ update_t_gen_column_args* argv = static_cast<update_t_gen_column_args*>(args);
+ const test & t = argv->t;
+ printer* p = argv->p;
+
+ // Check if printer is markdown printer
+ console_printer* md_printer = dynamic_cast<console_printer*>(p);
+ if (!md_printer) {
+ // For non-markdown printers, wait until test is completed
+ while (!t.test_completed) {
+ usleep(100000);
+ }
+ p->print_test(t);
+ return nullptr;
+ }
+
+ // For markdown printer, update continuously
+ p->print_test(t);
+ int last_t_gen = 0;
+ while (!t.test_completed) {
+ last_t_gen = t.t_gen;
+ // Move up to the previous line and clear it
+ printf("\033[A"); // Move up
+ printf("\033[2K"); // Clear the entire line
+
+ // Re-print the entire row with the updated t_gen value
+ p->print_test(t);
+
+ fflush(stdout);
+
+ usleep(100000); // sleep for 100ms (100,000 microseconds)
+ }
+
+ printf("\033[A"); // Move up
+ printf("\033[2K"); // Clear the entire line
+ p->print_test(t);
+ return nullptr;
+}
+
+std::string getUserConfirmation() {
+ std::string user_input;
+ printf("\nDo you want to submit your results to https://localscore.ai? The results will be public (y/n): ");
+ std::getline(std::cin, user_input);
+
+ // Convert to lowercase for case-insensitive comparison
+ std::transform(user_input.begin(), user_input.end(), user_input.begin(), ::tolower);
+ return user_input;
+}
+
+__attribute__((__constructor__(101))) static void init(void) {
+ FLAG_gpu = LLAMAFILE_GPU_AUTO;
+}
+
+static void warmup_run(llama_model *model, llama_context *ctx, cmd_params inst) {
+ printf("Warming up...... ");
+ int n_batch = inst.n_batch;
+ int n_processed = 0;
+ int n_prompt = inst.n_prompt;
+ int n_gen = inst.n_gen;
+
+ const int32_t n_vocab = llama_n_vocab(model);
+ std::vector<llama_token> tokens(n_batch);
+
+ llama_kv_cache_clear(ctx);
+
+ // warmup prompt
+ while (n_processed < n_prompt) {
+ int n_tokens = std::min(n_prompt - n_processed, n_batch);
+ tokens[0] = n_processed == 0 && llama_add_bos_token(model)
+ ? llama_token_bos(model)
+ : std::rand() % n_vocab;
+ for (int i = 1; i < n_tokens; i++) {
+ tokens[i] = std::rand() % n_vocab;
+ }
+ llama_decode(
+ ctx, llama_batch_get_one(tokens.data(), n_tokens, n_processed, 0));
+ n_processed += n_tokens;
+ }
+
+ llama_synchronize(ctx);
+
+ // warmup gen
+ llama_token token = llama_add_bos_token(model) ? llama_token_bos(model)
+ : std::rand() % n_vocab;
+ for (int i = 0; i < n_gen; i++) {
+ llama_decode(ctx, llama_batch_get_one(&token, 1, n_prompt + i, 0));
+ llama_synchronize(ctx);
+ token = std::rand() % n_vocab;
+ }
+
+ llama_free(ctx);
+
+ printf("Warmup complete.\n\n");
+}
+
+static bool submitBenchmarkResults(const std::string& req_payload, const cmd_params& params, int max_retries = 3) {
+ // Ask user for confirmation before sending the data
+ std::string user_cnf;
+ if (params.send_results == SEND_ASK) {
+ user_cnf = getUserConfirmation();
+ }
+
+ if (!(user_cnf == "yes" || user_cnf == "y" || params.send_results == SEND_YES)) {
+ printf("\nResults Not Submitted.\n");
+ return false;
+ }
+
+ if (params.verbose) {
+ printf("Submitting results...\n Payload: %s\n", req_payload.c_str());
+ } else {
+ printf("\nSubmitting results...\n");
+ }
+
+ // Implement retry with exponential backoff
+ for (int attempt = 0; attempt < max_retries; attempt++) {
+ if (attempt > 0) {
+ // Exponential backoff: wait 2^attempt seconds before retrying
+ int wait_time = (1 << attempt);
+ printf("Retry attempt %d of %d after %d seconds...\n", attempt + 1, max_retries, wait_time);
+ usleep(wait_time * 1000000);
+ }
+
+ try {
+ Response response = POST(BASE_LOCALSCORE_URL + "/api/results", req_payload, {
+ {"Content-Type", "application/json"}
+ });
+
+ if (response.status == 200) {
+ std::pair<Json::Status, Json> json = Json::parse(response.body);
+
+ if (json.first != Json::success) {
+ printf("Error parsing response json\n");
+ continue;
+ }
+ if (!json.second.isObject()) {
+ printf("Response json is not an object\n");
+ continue;
+ }
+
+ if (json.second["id"].isNumber()) {
+ printf("Result Link: %s/result/%d\n",
+ BASE_LOCALSCORE_URL.c_str(),
+ (int)json.second["id"].getNumber());
+ return true;
+ }
+ } else {
+ printf("Error submitting results to the public database. Status: %d\n", response.status);
+ if (attempt < max_retries - 1) {
+ continue;
+ }
+ }
+ } catch (const std::exception& e) {
+ printf("Error submitting results: %s\n", e.what());
+ if (attempt < max_retries - 1) {
+ continue;
+ }
+ }
+ }
+
+ printf("Failed to submit results after %d attempts\n", max_retries);
+ return false;
+}
+
+static void acceleratorSelector(cmd_params* params) {
+ if (FLAG_gpu >= 0 && llamafile_has_gpu()) {
+ if (llamafile_has_cuda()) {
+ int count = ggml_backend_cuda_get_device_count();
+ if (params->main_gpu == UINT_MAX) {
+ if (count == 1) {
+ params->main_gpu = 0;
+ } else {
+ fprintf(stderr, "\n\033[0;33mMultiple GPUs detected. Please select the main GPU to use:\n");
+ list_available_accelerators();
+ fprintf(stderr, "\n\033[0m");
+ unsigned int main_gpu;
+ while (true) {
+ fprintf(stderr, "Enter the number of the main GPU: ");
+ std::string input;
+ std::getline(std::cin, input);
+ std::istringstream iss(input);
+ if (iss >> main_gpu && main_gpu >= 0 && main_gpu < count) {
+ break;
+ }
+ fprintf(stderr, "Invalid GPU number. Please try again.\n");
+ }
+ params->main_gpu = main_gpu;
+ }
+ }
+ }
+ }
+}
+
+struct LocalScoreResultsSummary {
+ double avg_prompt_tps;
+ double avg_gen_tps;
+ double avg_ttft_ms;
+ double performance_score;
+};
+
+static LocalScoreResultsSummary getResultsSummary(Json data) {
+ LocalScoreResultsSummary rs = {
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ };
+
+ if (data["results"].isArray()) {
+ std::vector<Json> results = data["results"].getArray();
+
+ double total_prompt_tps = 0.0;
+ double total_gen_tps = 0.0;
+ double total_ttft_ms = 0.0;
+ int valid_count = 0;
+
+ for (const auto & result : results) {
+ if (result.isObject()) {
+ bool valid_entry = true;
+
+ // Check if all required fields exist and are numbers
+ if (!result.contains("prompt_tps") ||
+ !result.contains("gen_tps") ||
+ !result.contains("ttft_ms")) {
+ valid_entry = false;
+ } else {
+ // Get a non-const reference to access the values
+ auto& obj = const_cast<Json&>(result);
+ if (!obj["prompt_tps"].isNumber() ||
+ !obj["gen_tps"].isNumber() ||
+ !obj["ttft_ms"].isNumber()) {
+ valid_entry = false;
+ }
+ }
+
+ if (valid_entry) {
+ auto& obj = const_cast<Json&>(result);
+ total_prompt_tps += obj["prompt_tps"].getNumber();
+ total_gen_tps += obj["gen_tps"].getNumber();
+ total_ttft_ms += obj["ttft_ms"].getNumber();
+ valid_count++;
+ }
+ }
+ }
+
+ if (valid_count > 0) {
+ rs.avg_prompt_tps = total_prompt_tps / valid_count;
+ rs.avg_gen_tps = total_gen_tps / valid_count;
+ rs.avg_ttft_ms = total_ttft_ms / valid_count;
+
+ // calculate the geometric mean of the performance values for a score
+ double score = pow(rs.avg_prompt_tps * rs.avg_gen_tps * (1000 / rs.avg_ttft_ms), 1.0 / 3.0) * 10;
+
+ rs.performance_score = score;
+ }
+ }
+
+ return rs;
+}
+
+static void displayResults(LocalScoreResultsSummary results_summary) {
+ printf("\n\033[1;35m");
+ ascii_display::print_logo();
+ printf("\n");
+ ascii_display::printLargeNumber((int)results_summary.performance_score);
+ printf("\033[0m\n");
+ printf("\033[32mToken Generation: \t \033[1;32m%.2f\033[0m \033[3;32mtok/s\033[0m\n", results_summary.avg_gen_tps);
+ printf("\033[36mPrompt Processing: \t \033[1;36m%.2f\033[0m \033[3;36mtok/s\033[0m\n", results_summary.avg_prompt_tps);
+ printf("\033[33mTime to First Token:\t \033[1;33m%.2f\033[0m \033[3;33mms\033[0m\n", results_summary.avg_ttft_ms);
+ printf("\033[0m");
+}
+
+struct SystemData {
+ RuntimeInfo runtime;
+ SystemInfo sys;
+ AcceleratorInfo accelerator;
+};
+
+
+// Helper function implementations
+std::vector<test_config> get_baseline_test_configs() {
+ return {
+ {1024, 16}, // 64:1 title generation
+ {4096, 256}, // 16:1 content summarization
+ {2048, 256}, // 8:1 lots of code to fix
+ {2048, 768}, // 3:1 standard code chat
+ {1024, 1024}, // 1:1 code back and forth
+ {1280, 3072}, // 1:3 reasoning over code
+ {384, 1152}, // 1:3 code gen with back and forth
+ {64, 1024}, // 1:16 code gen/ideation
+ {16, 1536} // 1:96 QA, Storytelling, Reasoning
+ };
+}
+
+void setup_initial_environment(int* argc, char*** argv, cmd_params* params, SystemData* sys_data) {
+ LoadZipArgs(argc, argv);
+ setlocale(LC_CTYPE, "C.UTF-8");
+ *params = parse_cmd_params(*argc, *argv);
+ FLAGS_READY = true;
+ acceleratorSelector(params);
+ get_runtime_info(&sys_data->runtime);
+ get_sys_info(&sys_data->sys);
+ get_accelerator_info(&sys_data->accelerator, params);
+}
+
+void initialize_llama_backend(const cmd_params& params) {
+ if (!params.verbose) {
+ llama_log_set(llama_null_log_callback, NULL);
+ ggml_backend_metal_log_set_callback(llama_null_log_callback, NULL);
+ }
+ llama_backend_init();
+ llama_numa_init(params.numa);
+}
+
+llama_model* load_model(const cmd_params& params) {
+ printf("Loading model... ");
+ cmd_params inst = params;
+ inst.n_prompt = 1024;
+ inst.n_gen = 16;
+ llama_model* model = llama_load_model_from_file(inst.model.c_str(), inst.to_llama_mparams());
+ if (!model) {
+ fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, inst.model.c_str());
+ }
+ printf("Model loaded.\n");
+ return model;
+}
+
+std::unique_ptr<printer> create_printer(const cmd_params& params) {
+ std::unique_ptr<printer> p;
+ switch (params.output_format) {
+ case CSV: p.reset(new csv_printer()); break;
+ case JSON: p.reset(new json_printer()); break;
+ case CONSOLE: p.reset(new console_printer()); break;
+ default: assert(false); exit(1);
+ }
+ p->set_file_output(stdout);
+ return p;
+}
+
+void perform_warmup(llama_model* model, const cmd_params& params) {
+ cmd_params inst = params;
+ inst.n_prompt = 1024;
+ inst.n_gen = 16;
+
+ llama_context_params cparams = inst.to_llama_cparams();
+ cparams.n_ctx = inst.n_prompt + inst.n_gen;
+
+ llama_context* ctx = llama_new_context_with_model(model, cparams);
+ if (!ctx) {
+ fprintf(stderr, "%s: error: failed to create warmup context\n", __func__);
+ exit(1);
+ }
+ // ctx free happens in warmup
+ warmup_run(model, ctx, inst);
+}
+
+bool run_baseline_tests(const std::vector<test_config>& tests, llama_model* model,
+ const cmd_params& params, printer* p, PowerSampler* sampler,
+ json_printer* req_printer) {
+ for (const auto& test_cfg : tests) {
+ cmd_params inst = params;
+ inst.n_prompt = test_cfg.n_prompt;
+ inst.n_gen = test_cfg.n_gen;
+
+ llama_context_params cparams = inst.to_llama_cparams();
+ cparams.n_ctx = test_cfg.n_prompt + test_cfg.n_gen;
+
+ llama_context* ctx = llama_new_context_with_model(model, cparams);
+ if (!ctx) {
+ fprintf(stderr, "%s: error: failed to create context\n", __func__);
+ return false;
+ }
+
+ test t(inst, model, ctx, params.reps, sampler);
+
+ update_t_gen_column_args args = {t, p};
+ pthread_t update_thread;
+ if (int rc = pthread_create(&update_thread, NULL, update_t_gen_column, &args)) {
+ fprintf(stderr, "Error creating pthread: %d\n", rc);
+ return false;
+ }
+
+ t.run();
+ pthread_join(update_thread, NULL);
+ req_printer->print_test(t);
+
+ llama_free(ctx);
+ }
+ return true;
+}
+
+void process_and_submit_results(const std::string& req_payload, const cmd_params& params) {
+ auto [status, data] = Json::parse(req_payload);
+ if (status != Json::success || !data.isObject()) {
+ printf("Invalid JSON results\n");
+ exit(1);
+ }
+ LocalScoreResultsSummary rs = getResultsSummary(data);
+ displayResults(rs);
+
+ Json results_summary;
+ results_summary.setObject();
+ results_summary["avg_prompt_tps"] = rs.avg_prompt_tps;
+ results_summary["avg_gen_tps"] = rs.avg_gen_tps;
+ results_summary["avg_ttft_ms"] = rs.avg_ttft_ms;
+ results_summary["performance_score"] = rs.performance_score;
+
+ data["results_summary"] = results_summary;
+
+ const std::string payload = data.toString();
+
+ submitBenchmarkResults(payload, params);
+}
+
+int localscore_cli(int argc, char** argv) {
+ ShowCrashReports();
+
+ auto baseline_tests = get_baseline_test_configs();
+
+ cmd_params params;
+ SystemData sys_data;
+ setup_initial_environment(&argc, &argv, &params, &sys_data);
+
+ initialize_llama_backend(params);
+
+ llama_model* lmodel = load_model(params);
+ if (!lmodel) return 1;
+
+ ModelInfo model_info;
+ get_model_info(&model_info, lmodel);
+
+ std::string req_payload;
+ json_printer* req_printer = new json_printer();
+ req_printer->set_string_output(req_payload);
+ req_printer->print_header(params, sys_data.accelerator, sys_data.runtime, sys_data.sys, model_info);
+
+ auto p = create_printer(params);
+ PowerSampler* sampler = getPowerSampler(100, params.main_gpu);
+
+ perform_warmup(lmodel, params);
+
+ p->print_header(params, sys_data.accelerator, sys_data.runtime, sys_data.sys, model_info);
+
+ if (!run_baseline_tests(baseline_tests, lmodel, params, p.get(), sampler, req_printer)) {
+ llama_free_model(lmodel);
+ llama_backend_free();
+ return 1;
+ }
+
+ llama_free_model(lmodel);
+ p->print_footer();
+ req_printer->print_footer();
+
+ llama_backend_free();
+
+ process_and_submit_results(req_payload, params);
+ delete req_printer;
+
+ return 0;
+}
\ No newline at end of file
diff --git a/localscore/localscore.h b/localscore/localscore.h
new file mode 100644
index 00000000..65ad638a
--- /dev/null
+++ b/localscore/localscore.h
@@ -0,0 +1,3 @@
+#pragma once
+
+int localscore_cli(int argc, char **argv);
\ No newline at end of file
diff --git a/localscore/main.cpp b/localscore/main.cpp
new file mode 100644
index 00000000..ca58bf8f
--- /dev/null
+++ b/localscore/main.cpp
@@ -0,0 +1,5 @@
+#include "localscore.h"
+
+int main(int argc, char** argv) {
+ return localscore_cli(argc, argv);
+}
diff --git a/localscore/net.h b/localscore/net.h
new file mode 100644
index 00000000..4fb85a45
--- /dev/null
+++ b/localscore/net.h
@@ -0,0 +1,27 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "third_party/mbedtls/x509_crt.h"
+#include <ctime>
+
+namespace lf {
+
+mbedtls_x509_crt *sslroots(void);
+int socket(int, int, int, bool, const struct timeval *);
+
+} // namespace lf
diff --git a/localscore/nvml.cpp b/localscore/nvml.cpp
new file mode 100644
index 00000000..033eeebb
--- /dev/null
+++ b/localscore/nvml.cpp
@@ -0,0 +1,169 @@
+#include <cosmo.h>
+#include <dlfcn.h>
+#include <sys/stat.h>
+
+#include "nvml.h"
+#include "llama.cpp/common.h"
+
+static void *imp(void *lib, const char *sym) {
+ void *fun = cosmo_dlsym(lib, sym);
+ if (!fun)
+ tinylog(__func__, ": error: failed to import symbol: ", sym, "\n", NULL);
+ return fun;
+}
+
+static struct Nvml {
+ union {
+ int (*default_abi)(void);
+ int (__attribute__((__ms_abi__)) *windows_abi)(void);
+ } nvmlInit_v2;
+
+ union {
+ int (*default_abi)(unsigned int *deviceCount);
+ int (__attribute__((__ms_abi__)) *windows_abi)(unsigned int *deviceCount);
+ } nvmlDeviceGetCount_v2;
+
+ union {
+ int (*default_abi)(unsigned int index, void **device);
+ int (__attribute__((__ms_abi__)) *windows_abi)(unsigned int index, void **device);
+ } nvmlDeviceGetHandleByIndex_v2;
+
+ union {
+ int (*default_abi)(void *device, unsigned long long *energy);
+ int (__attribute__((__ms_abi__)) *windows_abi)(void *device, unsigned long long *energy);
+ } nvmlDeviceGetTotalEnergyConsumption;
+
+ union {
+ int (*default_abi)(void *device, unsigned int *power);
+ int (__attribute__((__ms_abi__)) *windows_abi)(void *device, unsigned int *power);
+ } nvmlDeviceGetPowerUsage;
+
+ union {
+ int (*default_abi)(void);
+ int (__attribute__((__ms_abi__)) *windows_abi)(void);
+ } nvmlShutdown;
+} nvml;
+
+template <typename UnionType>
+static void import_nvml_function(void *lib, const char *func_name, UnionType *member, bool *ok) {
+ using DefaultFuncType = decltype(UnionType::default_abi);
+ using WindowsFuncType = decltype(UnionType::windows_abi);
+
+ void *func_ptr = imp(lib, func_name);
+ if (!func_ptr) {
+ *ok = false;
+ return;
+ }
+
+ if (IsWindows()) {
+ member->windows_abi = reinterpret_cast<WindowsFuncType>(func_ptr);
+ } else {
+ member->default_abi = reinterpret_cast<DefaultFuncType>(func_ptr);
+ }
+
+ *ok &= true;
+}
+
+template <typename UnionType, typename... Args>
+auto nvml_function_call(UnionType &func_union, Args &&... args) -> decltype(func_union.default_abi(std::forward<Args>(args)...)) {
+ if (IsWindows()) {
+ return func_union.windows_abi(std::forward<Args>(args)...);
+ } else {
+ return func_union.default_abi(std::forward<Args>(args)...);
+ }
+}
+
+static bool FileExists(const char *path) {
+ struct stat st;
+ return !stat(path, &st);
+}
+
+static bool get_nvml_bin_path(char path[PATH_MAX]) {
+ char name[NAME_MAX];
+ if (IsWindows())
+ strlcpy(name, "nvml.dll", PATH_MAX);
+ else
+ strlcpy(name, "libnvidia-ml.so", PATH_MAX);
+
+ if (commandv(name, path, PATH_MAX)) {
+ printf("Found nvml on path: %s\n", path);
+ return true;
+ }
+
+ if (IsWindows()) {
+ const char *program_files = getenv("ProgramW6432");
+ if (!program_files) {
+ tinylog(__func__, ": note: $ProgramW6432 not set\n", NULL);
+ return false;
+ }
+ snprintf(path, PATH_MAX, "%s\\NVIDIA Corporation\\NVSMI\\%s", program_files, name);
+ printf("Attempting to load %s\n", path);
+ if (FileExists(path)) {
+ return true;
+ } else {
+ tinylog(__func__, ": note: %s does not exist\n", path);
+ snprintf(path, PATH_MAX, "C:\\Windows\\System32\\%s", name);
+ if (FileExists(path)) {
+ return true;
+ } else {
+ tinylog(__func__, ": note: %s does not exist\n", path);
+ return false;
+ }
+ }
+ } else {
+ strlcpy(path, name, PATH_MAX);
+ return true;
+ }
+}
+
+bool nvml_init() {
+ char dso[PATH_MAX];
+ if (!get_nvml_bin_path(dso)) {
+ tinylog(__func__, ": error: failed to find nvml library\n", NULL);
+ return false;
+ }
+
+ void *lib = cosmo_dlopen(dso, RTLD_LAZY);
+ bool ok = true;
+
+ // TODO we need a more robust way to import symbols and versions.
+ // this may end in a segfault currently.
+ import_nvml_function(lib, "nvmlInit_v2", &nvml.nvmlInit_v2, &ok);
+ import_nvml_function(lib, "nvmlDeviceGetCount_v2", &nvml.nvmlDeviceGetCount_v2, &ok);
+ import_nvml_function(lib, "nvmlDeviceGetHandleByIndex_v2", &nvml.nvmlDeviceGetHandleByIndex_v2, &ok);
+ import_nvml_function(lib, "nvmlDeviceGetTotalEnergyConsumption", &nvml.nvmlDeviceGetTotalEnergyConsumption, &ok);
+ import_nvml_function(lib, "nvmlDeviceGetPowerUsage", &nvml.nvmlDeviceGetPowerUsage, &ok);
+ import_nvml_function(lib, "nvmlShutdown", &nvml.nvmlShutdown, &ok);
+
+ if (!ok) {
+ tinylog(__func__, ": error: not all nvml symbols could be imported\n", NULL);
+ cosmo_dlclose(lib);
+ return false;
+ }
+
+ nvml_function_call(nvml.nvmlInit_v2);
+ unsigned int deviceCount;
+ nvml_function_call(nvml.nvmlDeviceGetCount_v2, &deviceCount);
+
+ return true;
+}
+
+bool nvml_get_device(nvmlDevice_t *device, unsigned int index) {
+ nvml_function_call(nvml.nvmlDeviceGetHandleByIndex_v2, index, device);
+ return true;
+}
+
+bool nvml_get_power_usage(nvmlDevice_t device, unsigned int *power) {
+ nvml_function_call(nvml.nvmlDeviceGetPowerUsage, device, power);
+ return true;
+}
+
+bool nvml_get_energy_consumption(nvmlDevice_t device, unsigned long long *energy) {
+ nvml_function_call(nvml.nvmlDeviceGetTotalEnergyConsumption, device, energy);
+ return true;
+}
+
+bool nvml_shutdown() {
+ nvml_function_call(nvml.nvmlShutdown);
+ return true;
+}
\ No newline at end of file
diff --git a/localscore/nvml.h b/localscore/nvml.h
new file mode 100644
index 00000000..d0852dfc
--- /dev/null
+++ b/localscore/nvml.h
@@ -0,0 +1,15 @@
+#pragma once
+
+typedef void* nvmlDevice_t;
+
+bool nvml_init();
+
+bool nvml_get_device(nvmlDevice_t *device, unsigned int index);
+
+bool nvml_get_energy_consumption(nvmlDevice_t device, unsigned long long *energy);
+
+bool nvml_get_power_usage(nvmlDevice_t device, unsigned int *power);
+
+bool nvml_get_memory_usage(nvmlDevice_t device, float *memory);
+
+bool nvml_shutdown();
\ No newline at end of file
diff --git a/localscore/powersampler.cpp b/localscore/powersampler.cpp
new file mode 100644
index 00000000..0f787fba
--- /dev/null
+++ b/localscore/powersampler.cpp
@@ -0,0 +1,240 @@
+#include "powersampler.h"
+#include <unistd.h>
+#include <cosmo.h>
+
+#include "llamafile/llamafile.h"
+#include "llama.cpp/ggml-metal.h"
+
+PowerSampler::PowerSampler(long sample_length_ms)
+ : sample_length_ms_(sample_length_ms), is_sampling_(false) {
+ pthread_mutex_init(&samples_mutex_, nullptr);
+}
+
+PowerSampler::~PowerSampler() {
+ if (is_sampling_) {
+ stop();
+ }
+ pthread_mutex_destroy(&samples_mutex_);
+}
+
+void PowerSampler::start() {
+ if (!is_sampling_) {
+ is_sampling_ = true;
+ samples_.clear();
+ sampling_start_time_ = timespec_real();
+ energy_consumed_start_ = getEnergyConsumed();
+
+ pthread_attr_t attr;
+ pthread_attr_init(&attr);
+ pthread_attr_setstacksize(&attr, 1*1024*1024); // set the stack size to 1MB
+ pthread_create(&sampling_thread_, &attr, sampling_thread_func, this);
+ }
+}
+
+power_sample_t PowerSampler::stop() {
+ power_sample_t result = {0.0};
+ if (is_sampling_) {
+ is_sampling_ = false;
+ sampling_end_time_ = timespec_real();
+ double energy_consumed_end = getEnergyConsumed();
+
+ long long sampling_time = timespec_tomillis(timespec_sub(sampling_end_time_, sampling_start_time_));
+ double energy_consumed = energy_consumed_end - energy_consumed_start_;
+ pthread_join(sampling_thread_, nullptr);
+
+ // average the samples
+ double total_milliwatts = 0;
+
+ if (samples_.size() > 1) {
+ for (int i = 0; i < samples_.size(); i++) {
+ total_milliwatts += samples_[i].power;
+ }
+ }
+
+ double avg_milliwatts = total_milliwatts / samples_.size();
+ double avg_watts = avg_milliwatts / 1e3;
+ double avg_watts_energy = energy_consumed / sampling_time;
+
+ if (FLAG_verbose) {
+ printf("Average power consumption from samples: %.2f mW, %.2f W\n", avg_milliwatts, avg_milliwatts / 1000);
+ printf("Total energy consumed: %.2f mJ, %.2fJ in %d ms\n", energy_consumed, energy_consumed / 1000, sampling_time);
+ printf("Average power from energy consumed: %.2f W \n", energy_consumed / sampling_time);
+ }
+
+ // pick the higher reading of the two
+ result.power = (avg_watts > avg_watts_energy) ? avg_watts : avg_watts_energy;
+ }
+
+ return result;
+}
+
+// this will return the latest sample in mw
+power_sample_t PowerSampler::getLatestSample() {
+ pthread_mutex_lock(&samples_mutex_);
+
+ if (samples_.empty()) {
+ pthread_mutex_unlock(&samples_mutex_);
+ return {0.0};
+ }
+
+ power_sample_t sample = samples_.back();
+ pthread_mutex_unlock(&samples_mutex_);
+
+ return sample;
+}
+
+void* PowerSampler::sampling_thread_func(void* arg) {
+ PowerSampler* sampler = static_cast<PowerSampler*>(arg);
+
+ while (sampler->is_sampling_) {
+ usleep(sampler->sample_length_ms_ * 1000); // Convert ms to microseconds
+ power_sample_t sample = sampler->sample();
+
+ pthread_mutex_lock(&sampler->samples_mutex_);
+ sampler->samples_.push_back(sample);
+ pthread_mutex_unlock(&sampler->samples_mutex_);
+
+ }
+
+ return nullptr;
+}
+
+NvidiaPowerSampler::NvidiaPowerSampler(long sample_length_ms, unsigned int main_gpu)
+ : PowerSampler(sample_length_ms) {
+ bool ok = nvml_init();
+ if (!ok) {
+ throw std::runtime_error("Failed to initialize NVML");
+ }
+
+ ok = nvml_get_device(&device_, main_gpu);
+ if (!ok) {
+ throw std::runtime_error("Failed to get NVML device");
+ }
+ }
+
+NvidiaPowerSampler::~NvidiaPowerSampler() {
+ nvml_shutdown();
+}
+
+power_sample_t NvidiaPowerSampler::sample() {
+ power_sample_t sample;
+ unsigned int mw;
+
+ if (!nvml_get_power_usage(device_, &mw)) {
+ // TODO return a bool instead? error?
+ }
+
+ sample.power = (double)mw;
+
+ return sample;
+}
+
+double NvidiaPowerSampler::getEnergyConsumed() {
+ unsigned long long mj;
+ if (!nvml_get_energy_consumption(device_, &mj)) {
+ return 0.0;
+ }
+ return (double)mj;
+}
+
+AMDPowerSampler::AMDPowerSampler(long sample_length_ms)
+ : PowerSampler(sample_length_ms) {
+ rsmi_init();
+ }
+
+AMDPowerSampler::~AMDPowerSampler() {
+ rsmi_shutdown();
+}
+
+power_sample_t AMDPowerSampler::sample() {
+ power_sample_t sample;
+
+ double power;
+ float vram;
+
+ if (!rsmi_get_power(&power)) { }
+
+ sample.power = power;
+
+ return sample;
+}
+
+double AMDPowerSampler::getEnergyConsumed() {
+ double uj;
+ if (!rsmi_get_energy_count(&uj)) {
+ return 0.0;
+ }
+ return uj;
+}
+
+ApplePowerSampler::ApplePowerSampler(long sample_length_ms)
+ : PowerSampler(sample_length_ms) {
+ bool ok = init_apple_mon();
+ if (!ok) {
+ throw std::runtime_error("Failed to initialize Apple Power Monitoring");
+ }
+ power_channel_ = am_get_power_channels();
+ sub_ = am_get_subscription(power_channel_);
+ last_sample_time_ = timespec_tomillis(timespec_real());
+ last_sample_mj_ = getEnergyConsumed();
+ metal_backend_ = ggml_backend_metal_init();
+ }
+
+ApplePowerSampler::~ApplePowerSampler() {
+ am_release(power_channel_);
+ am_release(sub_);
+}
+
+power_sample_t ApplePowerSampler::sample() {
+ long long time = timespec_tomillis(timespec_real());
+ double mj = getEnergyConsumed();
+
+ double power = (mj - last_sample_mj_) / (time - last_sample_time_);
+ // TODO this probably should be protected by a mutex
+ last_sample_time_ = time;
+ last_sample_mj_ = mj;
+
+ // convert to power in milliwatts
+ power_sample_t sample = {power * 1e3};
+
+ return sample;
+}
+
+double ApplePowerSampler::getEnergyConsumed() {
+ double mj = am_sample_to_millijoules(am_sample_power(sub_, power_channel_));
+ return mj;
+}
+
+DummyPowerSampler::DummyPowerSampler(long sample_length_ms)
+ : PowerSampler(sample_length_ms) {}
+
+power_sample_t DummyPowerSampler::sample() {
+ return {0.0};
+}
+
+double DummyPowerSampler::getEnergyConsumed() {
+ return 0.0;
+}
+
+PowerSampler* getPowerSampler(long sample_length_ms, unsigned int main_gpu) {
+ if (IsXnu()) {
+ return new ApplePowerSampler(sample_length_ms);
+ }
+ // else if (llamafile_has_gpu() && FLAG_gpu != LLAMAFILE_GPU_DISABLE) {
+ // if (llamafile_has_amd_gpu()) {
+ // // TODO change this to AMD power sampler when it works.
+ // return new DummyPowerSampler(sample_length_ms);
+ // } else if (llamafile_has_cuda()) {
+ // try {
+ // // TODO this will blow up on heterogeneous systems (AMD + NVIDIA hybrids probably)
+ // return new NvidiaPowerSampler(sample_length_ms, main_gpu);
+ // } catch (const std::exception& e) {
+ // // Log the error if needed
+ // printf("NVIDIA Power Monitoring Failed failed: %s\n", e.what());
+ // return new DummyPowerSampler(sample_length_ms);
+ // }
+ // }
+ // }
+
+ return new DummyPowerSampler(sample_length_ms);
+}
\ No newline at end of file
diff --git a/localscore/powersampler.h b/localscore/powersampler.h
new file mode 100644
index 00000000..13158c30
--- /dev/null
+++ b/localscore/powersampler.h
@@ -0,0 +1,94 @@
+#pragma once
+
+#include <pthread.h>
+#include <time.h>
+#include <vector>
+#include "nvml.h"
+#include "rsmi.h"
+#include "apple.h"
+#include "llama.cpp/ggml-backend-impl.h"
+
+typedef struct {
+ double power;
+} power_sample_t;
+
+struct PowerSampler {
+ // vars
+ long sample_length_ms_;
+
+ timespec sampling_start_time_;
+ timespec sampling_end_time_;
+ double energy_consumed_start_;
+
+ std::vector<power_sample_t> samples_;
+
+ bool is_sampling_;
+ pthread_t sampling_thread_;
+ mutable pthread_mutex_t samples_mutex_;
+
+ // funcs
+ PowerSampler(long sample_length_ms);
+ virtual ~PowerSampler();
+
+ void start();
+ power_sample_t stop();
+
+ power_sample_t getLatestSample();
+
+ // this returns the instantaneous power in microwatts
+ virtual power_sample_t sample() = 0;
+
+ // this returns the energy consumed in millijoules
+ virtual double getEnergyConsumed() = 0;
+
+private:
+ static void* sampling_thread_func(void* arg);
+};
+
+struct NvidiaPowerSampler : public PowerSampler {
+ nvmlDevice_t device_;
+ unsigned long long start_joules_;
+ unsigned long long end_joules_;
+
+ NvidiaPowerSampler(long sample_length_ms, unsigned int index);
+ ~NvidiaPowerSampler() override;
+
+protected:
+ power_sample_t sample() override;
+ double getEnergyConsumed() override;
+};
+
+struct AMDPowerSampler : public PowerSampler {
+ AMDPowerSampler(long sample_length_ms);
+ ~AMDPowerSampler() override;
+
+protected:
+ power_sample_t sample() override;
+ double getEnergyConsumed() override;
+};
+
+struct ApplePowerSampler : public PowerSampler {
+ ggml_backend_t metal_backend_;
+ CFMutableDictionaryRef power_channel_;
+ IOReportSubscriptionRef sub_;
+ long long last_sample_time_;
+ double last_sample_mj_;
+
+ ApplePowerSampler(long sample_length_ms);
+ ~ApplePowerSampler() override;
+
+protected:
+ power_sample_t sample() override;
+ double getEnergyConsumed() override;
+};
+
+struct DummyPowerSampler : public PowerSampler {
+ DummyPowerSampler(long sample_length_ms);
+ ~DummyPowerSampler() override {}
+
+protected:
+ power_sample_t sample() override;
+ double getEnergyConsumed() override;
+};
+
+PowerSampler* getPowerSampler(long sample_length_ms, unsigned int index);
diff --git a/localscore/printer.cpp b/localscore/printer.cpp
new file mode 100644
index 00000000..89de78de
--- /dev/null
+++ b/localscore/printer.cpp
@@ -0,0 +1,410 @@
+#include <cassert>
+
+#include "printer.h"
+#include "utils.h"
+
+#include "llama.cpp/string.h"
+
+FileWriter::FileWriter(FILE* f): fout(f) {}
+
+void FileWriter::write(const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ vfprintf(fout, format, args);
+ va_end(args);
+}
+
+void FileWriter::flush() {
+ fflush(fout);
+}
+
+StringWriter::StringWriter(std::string& str) : output(str) {}
+
+void StringWriter::write(const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ char tmp[1024];
+ vsnprintf(tmp, sizeof(tmp), format, args);
+ output += tmp;
+ va_end(args);
+}
+
+void StringWriter::flush() {}
+
+std::string csv_printer::escape_csv(const std::string& field) {
+ std::string escaped = "\"";
+ for (auto c : field) {
+ if (c == '"') {
+ escaped += "\"";
+ }
+ escaped += c;
+ }
+ escaped += "\"";
+ return escaped;
+}
+
+void csv_printer::print_header(const cmd_params& params,
+ AcceleratorInfo accelerator_info,
+ RuntimeInfo runtime_info,
+ SystemInfo sys_info,
+ ModelInfo model_info) {
+ std::vector<std::string> fields = test::get_fields();
+ writer->write("%s\n", utils::join(fields, ",").c_str());
+ (void) params;
+}
+
+void csv_printer::print_test(const test& t) {
+ std::vector<std::string> values = t.get_values();
+ std::transform(values.begin(), values.end(), values.begin(), escape_csv);
+ writer->write("%s\n", utils::join(values, ",").c_str());
+}
+
+std::string json_printer::escape_json(const std::string & value) {
+ std::string escaped;
+ for (auto c : value) {
+ if (c == '"') {
+ escaped += "\\\"";
+ } else if (c == '\\') {
+ escaped += "\\\\";
+ } else if (c <= 0x1f) {
+ char buf[8];
+ snprintf(buf, sizeof(buf), "\\u%04x", c);
+ escaped += buf;
+ } else {
+ escaped += c;
+ }
+ }
+ return escaped;
+}
+
+std::string json_printer::format_value(const std::string & field, const std::string & value) {
+ switch (test::get_field_type(field)) {
+ case test::STRING:
+ return "\"" + escape_json(value) + "\"";
+ case test::BOOL:
+ return value == "0" ? "false" : "true";
+ default:
+ return value;
+ }
+}
+
+void json_printer::print_header(const cmd_params & params, AcceleratorInfo accelerator_info, RuntimeInfo runtime_info, SystemInfo sys_info, ModelInfo model_info) {
+ // TODO we really should use jart's JSON.
+ writer->write("{\n");
+
+ // Print RuntimeInfo object
+ writer->write(" \"runtime_info\": {\n");
+ writer->write(" \"name\": \"%s\",\n", "llamafile");
+ writer->write(" \"version\": \"%s\",\n", runtime_info.llamafile_version);
+ writer->write(" \"commit\": \"%s\"\n", runtime_info.llama_commit);
+ writer->write(" },\n");
+
+ // Print SystemInfo object
+ writer->write(" \"system_info\": {\n");
+ writer->write(" \"cpu_name\": \"%s\",\n", sys_info.cpu);
+ writer->write(" \"cpu_arch\": \"%s\",\n", sys_info.system_architecture);
+ writer->write(" \"ram_gb\": %.2f,\n", sys_info.ram_gb);
+ writer->write(" \"kernel_type\": \"%s\",\n", sys_info.kernel_type);
+ writer->write(" \"kernel_release\": \"%s\",\n", sys_info.kernel_release);
+ writer->write(" \"version\": \"%s\"\n", sys_info.version);
+ writer->write(" },\n");
+
+ // Print GPUInfo object
+ writer->write(" \"accelerator_info\": {\n");
+ writer->write(" \"name\": \"%s\",\n", accelerator_info.name);
+ writer->write(" \"manufacturer\": \"%s\",\n", accelerator_info.manufacturer);
+ writer->write(" \"memory_gb\": %.2f,\n", accelerator_info.total_memory_gb);
+ writer->write(" \"type\": \"%s\"\n", (FLAG_gpu >= 0 && llamafile_has_gpu()) ? "GPU" : "CPU");
+ writer->write(" },\n");
+
+ // Start the results array
+ writer->write(" \"results\": [\n");
+
+ (void) params;
+}
+
+void json_printer::print_fields(const std::vector<std::string> & fields, const std::vector<std::string> & values) {
+ assert(fields.size() == values.size());
+ for (size_t i = 0; i < fields.size(); i++) {
+ writer->write(" \"%s\": %s,\n", fields.at(i).c_str(), format_value(fields.at(i), values.at(i)).c_str());
+ }
+}
+
+void json_printer::print_test(const test & t) {
+ if (first) {
+ first = false;
+ } else {
+ writer->write(",\n");
+ }
+ writer->write(" {\n");
+ print_fields(test::get_fields(), t.get_values());
+ writer->write(" \"samples_ns\": [ %s ]\n", utils::join(t.get_samples_ns(), ", ").c_str());
+ writer->write(" }");
+ writer->flush();
+}
+
+void json_printer::print_footer() {
+ writer->write("\n ]\n");
+ writer->write("}");
+}
+
+int console_printer::get_field_width(const std::string & field) {
+ if (field == "model") {
+ return -30;
+ }
+ if (field == "t/s") {
+ return 15;
+ }
+ if (field == "cpu_info") {
+ return test::cpu_info.size();
+ }
+ if (field == "model_filename") {
+ return 40;
+ }
+ if (field == "size" || field == "params") {
+ return 10;
+ }
+ if (field == "n_gpu_layers") {
+ return 3;
+ }
+ if (field == "test") {
+ return 13;
+ }
+
+ int width = std::max((int)field.length(), 10);
+
+ if (test::get_field_type(field) == test::STRING) {
+ return -width;
+ }
+ return width;
+}
+
+std::string console_printer::get_field_display_name(const std::string & field) {
+ if (field == "n_gpu_layers") {
+ return "ngl";
+ }
+ if (field == "split_mode") {
+ return "sm";
+ }
+ if (field == "n_threads") {
+ return "threads";
+ }
+ if (field == "no_kv_offload") {
+ return "nkvo";
+ }
+ if (field == "flash_attn") {
+ return "fa";
+ }
+ if (field == "use_mmap") {
+ return "mmap";
+ }
+ if (field == "embeddings") {
+ return "embd";
+ }
+ if (field == "tensor_split") {
+ return "ts";
+ }
+ return field;
+}
+
+int console_printer::calculate_total_width() const {
+ int total_width = 0;
+ for (const auto & field : fields) {
+ int width = get_field_width(field);
+ if (width < 0) {
+ width = std::abs(width);
+ }
+ total_width += width;
+ }
+ total_width += fields.size() * 3 + 1;
+ return total_width;
+}
+
+void console_printer::print_header(const cmd_params & params, AcceleratorInfo accelerator_info, RuntimeInfo runtime_info, SystemInfo sys_info, ModelInfo model_info) {
+ fields.emplace_back("test");
+ fields.emplace_back("run number");
+ fields.emplace_back("avg time");
+ // fields.emplace_back("power");
+ fields.emplace_back("tokens processed");
+ fields.emplace_back("pp t/s");
+ fields.emplace_back("tg t/s");
+ fields.emplace_back("ttft");
+
+ int total_width = calculate_total_width();
+
+ std::string border(total_width, '-');
+ border[0] = '+';
+ border[total_width-1] = '+';
+ writer->write("%s\n", border.c_str());
+
+ // Create the GPU info string
+ char gpu_info_str[256];
+ int content_length = snprintf(gpu_info_str, sizeof(gpu_info_str),
+ "%s - %.1f GiB",
+ accelerator_info.name,
+ accelerator_info.total_memory_gb);
+
+ // Calculate left and right padding to properly center the text
+ int left_padding = (total_width - 2 - content_length) / 2;
+ int right_padding = total_width - 2 - content_length - left_padding;
+
+ // Print the GPU info with correct padding
+ writer->write("|%*s%s%*s|\n",
+ left_padding, "",
+ gpu_info_str,
+ right_padding, "");
+
+ // Create the model info string
+ char model_info_str[256];
+ content_length = snprintf(model_info_str, sizeof(model_info_str),
+ "%s - %s",
+ model_info.name,
+ model_info.quant);
+
+ // Calculate left and right padding separately
+ left_padding = (total_width - 2 - content_length) / 2;
+ right_padding = total_width - 2 - content_length - left_padding;
+
+ // Print the model info with correct padding
+ writer->write("|%*s%s%*s|\n",
+ left_padding, "",
+ model_info_str,
+ right_padding, "");
+
+ writer->write("%s\n", border.c_str());
+
+ writer->write("|");
+ for (const auto & field : fields) {
+ writer->write(" %*s |", get_field_width(field), get_field_display_name(field).c_str());
+ }
+ writer->write("\n");
+ writer->write("|");
+ for (const auto & field : fields) {
+ int width = get_field_width(field);
+ writer->write(" %s |", std::string(std::abs(width), '-').c_str());
+ }
+ writer->write("\n");
+}
+
+void console_printer::print_test(const test & t) {
+ std::map<std::string, std::string> vmap = t.get_map();
+
+ // float power = t.get_power();
+
+ writer->write("|");
+ for (const auto & field : fields) {
+ std::string value;
+ char buf[128];
+ if (field == "model") {
+ value = t.model_type;
+ } else if (field == "size") {
+ if (t.model_size < 1024*1024*1024) {
+ snprintf(buf, sizeof(buf), "%.2f MiB", t.model_size / 1024.0 / 1024.0);
+ } else {
+ snprintf(buf, sizeof(buf), "%.2f GiB", t.model_size / 1024.0 / 1024.0 / 1024.0);
+ }
+ value = buf;
+ } else if (field == "params") {
+ snprintf(buf, sizeof(buf), "%ld", t.model_n_params);
+ // if (t.model_n_params < 1000*1000*1000) {
+ // snprintf(buf, sizeof(buf), "%.2f M", t.model_n_params / 1e6);
+ // } else {
+ // snprintf(buf, sizeof(buf), "%.2f B", t.model_n_params / 1e9);
+ // }
+ value = buf;
+ } else if (field == "backend") {
+ value = test::get_backend();
+ } else if (field == "run number") {
+ snprintf(buf, sizeof(buf), "%d/%d", t.curr_run + 1, t.reps);
+ value = buf;
+ } else if (field == "test") {
+ value = t.name;
+ } else if (field == "pp t/s") {
+ snprintf(buf, sizeof(buf), "%.2f", t.avg_ts(PROMPT_TPS));
+
+ value = buf;
+ } else if (field == "tg t/s") {
+ snprintf(buf, sizeof(buf), "%.2f", t.avg_ts(GEN_TPS));
+ if (!t.gen_intervals.empty() && t.curr_run < t.gen_intervals.size()) {
+ time_interval curr_interval = t.gen_intervals[t.curr_run];
+
+ if (curr_interval.end == 0) {
+ // get the live tps instead of avg
+ uint64_t elapsed_ns = utils::get_time_ns() - curr_interval.start;
+ float elapsed_s = elapsed_ns / 1e9;
+ float tps = t.t_gen / elapsed_s;
+ snprintf(buf, sizeof(buf), "%.2f", tps);
+ }
+ }
+
+ value = buf;
+ } else if (field == "tokens processed") {
+ int num_generated = t.t_gen + (t.curr_run * t.n_gen);
+ int num_processed = t.t_processed + (t.curr_run * t.n_prompt);
+
+ snprintf(buf, sizeof(buf), "%d / %d", num_generated + num_processed, (t.n_gen * t.reps) + (t.n_prompt * t.reps));
+
+ value = buf;
+ } else if (field == "pp t/s/watt") {
+ snprintf(buf, sizeof(buf), "%.4f", t.get_tps_watt(PROMPT_TPS));
+
+ value = buf;
+ } else if (field == "tg t/s/watt") {
+ snprintf(buf, sizeof(buf), "%.4f", t.get_tps_watt(GEN_TPS));
+
+ value = buf;
+ } else if (field == "ttft") {
+ float ttft = t.ttft() / 1e6;
+
+ if (ttft < 1000) {
+ snprintf(buf, sizeof(buf), "%.2f ms", ttft);
+ } else {
+ snprintf(buf, sizeof(buf), "%.2f s", ttft / 1e3);
+ }
+
+ value = buf;
+ } else if (field == "power") {
+ if (t.monitor_result.power > 0) {
+ snprintf(buf, sizeof(buf), "%.2f W", t.monitor_result.power);
+ value = buf;
+ } else {
+ // read instant power
+ power_sample_t sample = t.pwr_sampler->getLatestSample();
+ snprintf(buf, sizeof(buf), "%.2f W", sample.power / 1e3);
+ }
+
+ value = buf;
+ } else if (field == "avg time") {
+ float avg_ms = t.avg_ns() / 1e6;
+
+ if (avg_ms < 1000) {
+ snprintf(buf, sizeof(buf), "%.2f ms", avg_ms);
+ } else {
+ snprintf(buf, sizeof(buf), "%.2f s", avg_ms / 1e3);
+ }
+
+ value = buf;
+ } else if (vmap.find(field) != vmap.end()) {
+ value = replace_all(replace_all(vmap.at(field), ".gguf", ""), ".llamafile", ""); // [jart]
+ } else {
+ assert(false);
+ exit(1);
+ }
+
+ int width = get_field_width(field);
+ // if (field == "t/s") { // [jart]
+ // // HACK: the utf-8 character is 2 bytes
+ // width += 1;
+ // }
+ writer->write(" %*s |", width, value.c_str());
+ }
+ writer->write("\n");
+}
+
+void console_printer::print_footer() {
+ int total_width = calculate_total_width();
+ std::string border(total_width, '-');
+ border[0] = '+';
+ border[total_width-1] = '+';
+ writer->write("%s\n", border.c_str());
+}
\ No newline at end of file
diff --git a/localscore/printer.h b/localscore/printer.h
new file mode 100644
index 00000000..d725f62f
--- /dev/null
+++ b/localscore/printer.h
@@ -0,0 +1,99 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <cstdio>
+
+#include "system.h"
+#include "cmd.h"
+#include "benchmark.h"
+
+struct OutputWriter {
+ virtual ~OutputWriter() {};
+ virtual void write(const char* buf, ...) = 0;
+ virtual void flush() = 0;
+};
+
+struct FileWriter : public OutputWriter {
+ FILE* fout;
+
+ FileWriter(FILE* f);
+ void write(const char* format, ...) override;
+ void flush() override;
+};
+
+struct StringWriter : public OutputWriter {
+ std::string& output;
+
+ StringWriter(std::string& str);
+ void write(const char* format, ...) override;
+ void flush() override;
+};
+
+struct printer {
+ virtual ~printer() {}
+
+ std::unique_ptr<OutputWriter> writer;
+
+ void set_file_output(FILE* fout) {
+ writer = std::make_unique<FileWriter>(fout);
+ }
+
+ void set_string_output(std::string& output) {
+ writer = std::make_unique<StringWriter>(output);
+ }
+
+ virtual void print_header(const cmd_params & params, AcceleratorInfo accelerator_info, RuntimeInfo runtime_info, SystemInfo sys_info, ModelInfo model_info) { (void) params; }
+ virtual void print_test(const test & t) = 0;
+ virtual void print_footer() { }
+};
+
+struct csv_printer : public printer {
+ static std::string escape_csv(const std::string& field);
+
+ void print_header(const cmd_params& params,
+ AcceleratorInfo accelerator_info,
+ RuntimeInfo runtime_info,
+ SystemInfo sys_info,
+ ModelInfo model_info) override;
+
+ void print_test(const test& t) override;
+};
+
+struct json_printer : public printer {
+ bool first = true;
+
+ static std::string escape_json(const std::string& value);
+ static std::string format_value(const std::string& field, const std::string& value);
+
+ void print_header(const cmd_params& params,
+ AcceleratorInfo accelerator_info,
+ RuntimeInfo runtime_info,
+ SystemInfo sys_info,
+ ModelInfo model_info) override;
+
+ void print_test(const test& t) override;
+ void print_footer() override;
+
+private:
+ void print_fields(const std::vector<std::string>& fields,
+ const std::vector<std::string>& values);
+};
+
+struct console_printer : public printer {
+ std::vector<std::string> fields;
+
+ static int get_field_width(const std::string& field);
+ static std::string get_field_display_name(const std::string& field);
+
+ int calculate_total_width() const;
+
+ void print_header(const cmd_params& params,
+ AcceleratorInfo accelerator_info,
+ RuntimeInfo runtime_info,
+ SystemInfo sys_info,
+ ModelInfo model_info) override;
+ void print_test(const test& t) override;
+ void print_footer() override;
+};
+
diff --git a/localscore/rsmi.cpp b/localscore/rsmi.cpp
new file mode 100644
index 00000000..062d2aeb
--- /dev/null
+++ b/localscore/rsmi.cpp
@@ -0,0 +1,117 @@
+#include <cosmo.h>
+#include <dlfcn.h>
+
+#include "rsmi.h"
+#include "llama.cpp/common.h"
+
+#define IMPORT_RSMI_FUNCTION(func_name, func_type) \
+ ok &= !!(rsmi.func_name = (func_type)(imp(lib, #func_name)))
+
+#define RSMI_FUNCTION_CALL(func_name, error_msg, ...) \
+ do { \
+ if (!rsmi.func_name) { \
+ tinylog(__func__, ": error: " #func_name " not imported\n", NULL); \
+ return false; \
+ } \
+ int status = rsmi.func_name(__VA_ARGS__); \
+ if (status != 0) { \
+ tinylog(__func__, ": error: " error_msg "\n", NULL); \
+ return false; \
+ } \
+ } while(0)
+
+static void *imp(void *lib, const char *sym) {
+ void *fun = cosmo_dlsym(lib, sym);
+ if (!fun)
+ tinylog(__func__, ": error: failed to import symbol: ", sym, "\n", NULL);
+ return fun;
+}
+
+typedef enum {
+ RSMI_AVERAGE_POWER = 0, //!< Average Power
+ RSMI_CURRENT_POWER, //!< Current / Instant Power
+ RSMI_INVALID_POWER = 0xFFFFFFFF //!< Invalid / Undetected Power
+} RSMI_POWER_TYPE;
+
+static struct Rsmi {
+ int (*rsmi_init)(uint64_t init_flags);
+ int (*rsmi_num_monitor_devices)(uint32_t *num_devices);
+ int (*rsmi_dev_id_get)(uint32_t dv_ind, uint16_t *id);
+ int (*rsmi_dev_power_get)(uint32_t dv_ind, uint64_t *power, RSMI_POWER_TYPE *type);
+ int (*rsmi_dev_current_socket_power_get)(uint32_t dv_ind, uint64_t *power); // in uW
+ int (*rsmi_dev_power_ave_get)(uint32_t dv_ind, uint32_t sensor_ind, uint64_t *power);
+ int (*rsmi_dev_energy_count_get)(uint32_t dv_ind, uint64_t *power, float *counter_resolution, uint64_t *timestamp);
+ int (*rsmi_dev_memory_usage_get)(uint32_t dv_ind, int mem_type, uint64_t *used);
+ int (*rsmi_shut_down)(void);
+} rsmi;
+
+bool rsmi_init() {
+ void *lib = cosmo_dlopen("/opt/rocm/lib/librocm_smi64.so", RTLD_NOW);
+ bool ok = true;
+
+ IMPORT_RSMI_FUNCTION(rsmi_init, int (*)(uint64_t));
+ IMPORT_RSMI_FUNCTION(rsmi_num_monitor_devices, int (*)(uint32_t*));
+ IMPORT_RSMI_FUNCTION(rsmi_dev_id_get, int (*)(uint32_t, uint16_t*));
+ IMPORT_RSMI_FUNCTION(rsmi_dev_power_get, int (*)(uint32_t, uint64_t*, RSMI_POWER_TYPE*));
+ IMPORT_RSMI_FUNCTION(rsmi_dev_current_socket_power_get, int (*)(uint32_t, uint64_t*));
+ IMPORT_RSMI_FUNCTION(rsmi_dev_power_ave_get, int (*)(uint32_t, uint32_t, uint64_t*));
+ IMPORT_RSMI_FUNCTION(rsmi_dev_energy_count_get, int (*)(uint32_t, uint64_t*, float*, uint64_t*));
+ IMPORT_RSMI_FUNCTION(rsmi_dev_memory_usage_get, int (*)(uint32_t, int, uint64_t*));
+ IMPORT_RSMI_FUNCTION(rsmi_shut_down, int (*)(void));
+
+ if (!ok) {
+ tinylog(__func__, ": error: not all rocm smi symbols could be imported\n", NULL);
+ cosmo_dlclose(lib);
+ return false;
+ }
+
+ RSMI_FUNCTION_CALL(rsmi_init, "failed to initialize ROCm SMI", 0);
+ return true;
+}
+
+bool rsmi_get_avg_power(double *power) {
+ uint64_t power_val;
+ RSMI_FUNCTION_CALL(rsmi_dev_power_ave_get, "failed to get average power", 0, 0, &power_val);
+ *power = (double)power_val;
+ return true;
+}
+
+bool rsmi_get_power(double *power) {
+ uint64_t power_val;
+ RSMI_POWER_TYPE type;
+
+ RSMI_FUNCTION_CALL(rsmi_dev_power_get, "failed to get power", 0, &power_val, &type);
+ // Convert microwatts to milliwatts
+ *power = (double)power_val / 1000.0;
+ return true;
+}
+
+bool rsmi_get_energy_count(double *energy) {
+ uint64_t power;
+ float counter_resolution;
+ uint64_t timestamp;
+ RSMI_FUNCTION_CALL(rsmi_dev_energy_count_get, "failed to get energy count", 0, &power, &counter_resolution, &timestamp);
+ // Convert microjoules to millijoules
+ *energy = (double)(power * counter_resolution) / 1000.0;
+ return true;
+}
+
+bool rsmi_get_power_instant(double *power) {
+ uint64_t power_val;
+ RSMI_FUNCTION_CALL(rsmi_dev_current_socket_power_get, "failed to get instant power", 0, &power_val);
+ *power = (double)power_val;
+ return true;
+}
+
+bool rsmi_get_memory_usage(float *memory) {
+ uint64_t used;
+ // this is device 0 and memory type 0 (RSMI_MEM_TYPE_VRAM)
+ RSMI_FUNCTION_CALL(rsmi_dev_memory_usage_get, "failed to get memory usage", 0, 0, &used);
+ *memory = (float)used / 1024.0 / 1024.0;
+ return true;
+}
+
+bool rsmi_shutdown() {
+ RSMI_FUNCTION_CALL(rsmi_shut_down, "failed to shutdown ROCm SMI");
+ return true;
+}
\ No newline at end of file
diff --git a/localscore/rsmi.h b/localscore/rsmi.h
new file mode 100644
index 00000000..421cffad
--- /dev/null
+++ b/localscore/rsmi.h
@@ -0,0 +1,13 @@
+bool rsmi_init();
+
+bool rsmi_get_power(double *power);
+
+bool rsmi_get_power_instant(double *power);
+
+bool rsmi_get_avg_power(double *power);
+
+bool rsmi_get_energy_count(double *energy);
+
+bool rsmi_get_memory_usage(float *memory);
+
+bool rsmi_shutdown();
\ No newline at end of file
diff --git a/localscore/system.cpp b/localscore/system.cpp
new file mode 100644
index 00000000..2b0e040d
--- /dev/null
+++ b/localscore/system.cpp
@@ -0,0 +1,343 @@
+#include "system.h"
+#include <cstdio>
+#include <sys/utsname.h>
+#include <sys/sysinfo.h>
+#include <cosmo.h>
+#include <iostream>
+#include <string>
+#include <sstream>
+#include "llama.cpp/string.h"
+
+#include "cmd.h"
+#include "utils.h"
+
+#include "llama.cpp/ggml-metal.h"
+#include "llama.cpp/ggml-cuda.h"
+#include "llama.cpp/common.h"
+
+#include <libc/intrin/x86.h>
+
+#ifdef __x86_64__
+void cpuid(unsigned leaf, unsigned subleaf, unsigned *info) {
+ asm("movq\t%%rbx,%%rsi\n\t"
+ "cpuid\n\t"
+ "xchgq\t%%rbx,%%rsi"
+ : "=a"(info[0]), "=S"(info[1]), "=c"(info[2]), "=d"(info[3])
+ : "0"(leaf), "2"(subleaf));
+}
+
+// TODO implement an arm version as well
+char* get_cpu_manufacturer(void) {
+ union {
+ char str[13]; // 12 chars + null terminator
+ unsigned reg[4]; // For the 4 registers (EAX, EBX, ECX, EDX)
+ } u = {0}; // Initialize to zero
+
+ // Get manufacturer ID with leaf 0
+ cpuid(0, 0, u.reg);
+
+ // Rearrange the registers to get the correct string
+ // The manufacturer string is in EBX,EDX,ECX order
+ unsigned temp = u.reg[1]; // Save EBX
+ u.reg[0] = temp; // Move EBX to first position
+ u.reg[1] = u.reg[3]; // Move EDX to second position
+ u.reg[2] = u.reg[2]; // ECX stays in third position
+ u.reg[3] = 0; // Ensure null termination
+
+ const char* manufacturer = u.str;
+ if (strcmp(manufacturer, "AuthenticAMD") == 0) {
+ return strdup("AMD");
+ } else if (strcmp(manufacturer, "GenuineIntel") == 0) {
+ return strdup("Intel");
+ }
+
+ return strdup(manufacturer); // Return the original string if unknown
+}
+#endif // __x86_64__
+
+std::string get_cpu_info() { // [jart]
+ std::string id;
+
+#ifdef __x86_64__
+ union { // [jart]
+ char str[64];
+ unsigned reg[16];
+ } u = {0};
+ cpuid(0x80000002, 0, u.reg + 0*4);
+ cpuid(0x80000003, 0, u.reg + 1*4);
+ cpuid(0x80000004, 0, u.reg + 2*4);
+ int len = strlen(u.str);
+ while (len > 0 && u.str[len - 1] == ' ')
+ u.str[--len] = 0;
+ id = u.str;
+#else
+ if (IsLinux()) {
+ FILE * f = fopen("/proc/cpuinfo", "r");
+ if (f) {
+ char buf[1024];
+ while (fgets(buf, sizeof(buf), f)) {
+ if (!strncmp(buf, "model name", 10) ||
+ startswith(buf, "Model\t\t:")) { // e.g. raspi
+ char * p = strchr(buf, ':');
+ if (p) {
+ p++;
+ while (std::isspace(*p)) {
+ p++;
+ }
+ while (std::isspace(p[strlen(p) - 1])) {
+ p[strlen(p) - 1] = '\0';
+ }
+ id = p;
+ break;
+ }
+ }
+ }
+ fclose(f);
+ }
+ }
+ if (IsXnu()) {
+ // TODO we can also do something similar to https://github.com/vladkens/macmon/blob/main/src/sources.rs#L424
+ char cpu_name[128] = {0};
+ size_t size = sizeof(cpu_name);
+ if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1) {
+ id = cpu_name;
+ }
+
+ // Get number of performance cores on macos
+ int num_perf0_cpu;
+ size = sizeof(num_perf0_cpu);
+ if (sysctlbyname("hw.perflevel0.logicalcpu", &num_perf0_cpu, &size, NULL, 0) != -1) {
+ id += " ";
+ id += std::to_string(num_perf0_cpu);
+ id += "P";
+ }
+
+ // Get number of efficiency cores on macos
+ int num_perf1_cpu;
+ size = sizeof(num_perf1_cpu);
+ if (sysctlbyname("hw.perflevel1.logicalcpu", &num_perf1_cpu, &size, NULL, 0) != -1) {
+ id += "+";
+ id += std::to_string(num_perf1_cpu);
+ id += "E";
+ }
+
+ }
+#endif
+ id = replace_all(id, " 96-Cores", "");
+ id = replace_all(id, "(TM)", "");
+ id = replace_all(id, "(R)", "");
+
+ std::string march;
+#ifdef __x86_64__
+ if (__cpu_march(__cpu_model.__cpu_subtype))
+ march = __cpu_march(__cpu_model.__cpu_subtype);
+#else
+ // TODO. We can do this separately as part of 'features' or something
+ // long hwcap = getauxval(AT_HWCAP);
+ // if (hwcap & HWCAP_ASIMDHP)
+ // march += "+fp16";
+ // if (hwcap & HWCAP_ASIMDDP)
+ // march += "+dotprod";
+#endif
+
+ if (!march.empty()) {
+ bool empty = id.empty();
+ if (!empty)
+ id += " (";
+ id += march;
+ if (!empty)
+ id += ")";
+ }
+
+ return id;
+}
+
+void get_runtime_info(RuntimeInfo* info) {
+ if (info == NULL) return;
+
+ strncpy(info->llamafile_version, LLAMAFILE_VERSION_STRING, MAX_STRING_LENGTH - 1);
+ strncpy(info->llama_commit, LLAMA_COMMIT, MAX_STRING_LENGTH - 1);
+
+ fprintf(stderr, "\033[0;35m\n"); // Sets purple color
+ utils::print_centered(stderr, 70, '=', "\033[1mLocalScore Runtime Information\033[0;35m");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "%-20s \033[1m%s\033[22m\n", "llamafile version:", info->llamafile_version);
+ fprintf(stderr, "%-20s %s\n", "llama.cpp commit:", info->llama_commit);
+ fprintf(stderr, "\n======================================================================\n\n\033[0m");
+}
+
+double get_mem_gb() {
+ struct sysinfo si;
+ if (sysinfo(&si)) {
+ return 0.0;
+ }
+
+ return utils::round_to_decimal(si.totalram * si.mem_unit / 1073741824.0, 1);
+}
+
+void get_sys_info(SystemInfo* info) {
+ if (info == NULL) return;
+
+ struct utsname names;
+ if (uname(&names)) {
+ return;
+ }
+
+ utils::sanitize_string(info->kernel_type, names.sysname, MAX_STRING_LENGTH);
+ utils::sanitize_string(info->kernel_release, names.release, MAX_STRING_LENGTH);
+ // TODO on darwin we might want to get from systemprofiler SPSoftwareDataType os_version
+ utils::sanitize_string(info->version, names.version, MAX_STRING_LENGTH);
+ utils::sanitize_string(info->system_architecture, names.machine, MAX_STRING_LENGTH);
+
+ std::string cpu_info = get_cpu_info();
+ strncpy(info->cpu, cpu_info.c_str(), MAX_STRING_LENGTH - 1);
+
+ info->ram_gb = get_mem_gb();
+
+ utils::print_centered(stderr, 70, '=', "\033[1mSystem Information\033[0m");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "%-20s %s\n", "Kernel Type:", info->kernel_type);
+ fprintf(stderr, "%-20s %s\n", "Kernel Release:", info->kernel_release);
+ fprintf(stderr, "%-20s %s\n", "Version:", info->version);
+ fprintf(stderr, "%-20s %s\n", "System Architecture:", info->system_architecture);
+ fprintf(stderr, "%-20s %s\n", "CPU:", info->cpu);
+ fprintf(stderr, "%-20s %.1f GiB\n", "RAM:", info->ram_gb);
+ fprintf(stderr, "\n======================================================================\n\n");
+}
+
+void get_accelerator_info(AcceleratorInfo* info, cmd_params * params) {
+ if (info == NULL) return;
+
+ if (FLAG_gpu >= 0 && llamafile_has_gpu()) {
+ if (llamafile_has_cuda()) {
+ int count = ggml_backend_cuda_get_device_count();
+ for (int i = 0; i < count; i++) {
+ struct ggml_cuda_device_properties props;
+ ggml_backend_cuda_get_device_properties(i, &props);
+
+ if (params->verbose) {
+ printf("Raw GPU %d Memory %lld Bytes, %.2f GiB\n", i, props.totalGlobalMem, props.totalGlobalMem / 1073741824.0);
+ }
+
+ // TODO it would be much better to query NVML directly instead and similar for rocm
+ double rounded_memory_gb = utils::round_to_decimal(props.totalGlobalMem / 1073741824.0, 0);
+
+ if (i == params->main_gpu) {
+ strncpy(info->name, props.name, MAX_STRING_LENGTH - 1);
+
+ info->total_memory_gb = rounded_memory_gb;
+ info->core_count = props.multiProcessorCount;
+ info->capability = atof(props.compute);
+ strncpy(info->manufacturer, llamafile_has_amd_gpu() ? "AMD" : "NVIDIA", MAX_STRING_LENGTH - 1);
+ }
+
+ if (i == params->main_gpu) {
+ fprintf(stderr, "\033[0;32m"); // Sets green color
+ utils::print_centered(stderr, 70, '=', "\033[1mActive GPU (GPU %d) Information\033[0;32m", i);
+ fprintf(stderr, "\n");
+ } else {
+ fprintf(stderr, "\033[0;90m"); // Sets gray color
+ utils::print_centered(stderr, 70, '=', "GPU %d Information", i);
+ fprintf(stderr, "\n");
+ }
+
+ fprintf(stderr, "%-26s %s\n", "GPU Name:", props.name);
+ fprintf(stderr, "%-26s %.1f GiB\n", "VRAM:", rounded_memory_gb);
+ fprintf(stderr, "%-26s %d\n", "Streaming Multiprocessors:", props.multiProcessorCount);
+ fprintf(stderr, "%-26s %.1f\n", "CUDA Capability:", atof(props.compute));
+ fprintf(stderr, "\n======================================================================\n\n\033[0m");
+ }
+ }
+
+ if (llamafile_has_metal()) {
+ // TODO there is probably a cleaner way of doing this. we should only need to init once.
+ // this is probably the same issue why the other thing is init multiple time too
+ struct ggml_metal_device_properties props;
+
+ std::string command = "system_profiler SPDisplaysDataType | grep \"Total Number of Cores:\" | awk '{print $5}'";
+ std::string num_cores = utils::exec(command.c_str());
+ props.core_count = std::stoi(num_cores);
+
+ // Remove any trailing newline
+ if (!num_cores.empty() && num_cores[num_cores.length()-1] == '\n') {
+ num_cores.erase(num_cores.length()-1);
+ }
+
+ ggml_backend_t result = ggml_backend_metal_init();
+
+ ggml_backend_metal_get_device_properties(result, &props);
+
+ std::string cpu_info = get_cpu_info();
+ cpu_info += "+" + num_cores + "GPU";
+ strncpy(info->name, cpu_info.c_str(), MAX_STRING_LENGTH - 1);
+ info->total_memory_gb = get_mem_gb();
+ info->core_count = props.core_count;
+ info->capability = props.metal_version;
+ strncpy(info->manufacturer, "Apple", MAX_STRING_LENGTH - 1);
+
+
+ fprintf(stderr, "\033[0;32m===== GPU information =====\n\n");
+ fprintf(stderr, "%-26s %s\n", "GPU Name:", props.name);
+ fprintf(stderr, "%-26s %.1f GiB\n", "VRAM:", info->total_memory_gb);
+ fprintf(stderr, "%-26s %d\n", "Core Count:", props.core_count);
+ fprintf(stderr, "%-26s %d\n", "Metal Version:", props.metal_version);
+ fprintf(stderr, "%-26s %d\n", "GPU Family:", props.gpu_family);
+ fprintf(stderr, "%-26s %d\n", "Common GPU Family:", props.gpu_family_common);
+ fprintf(stderr, "\n======================================================================\n\n\033[0m");
+ }
+ } else {
+ #ifdef __x86_64__
+ strncpy(info->manufacturer, get_cpu_manufacturer(), MAX_STRING_LENGTH - 1);
+ #else
+ if IsXnu() {
+ strncpy(info->manufacturer, "Apple", MAX_STRING_LENGTH - 1);
+ } else {
+ strncpy(info->manufacturer, "Unknown", MAX_STRING_LENGTH - 1);
+ }
+ #endif
+ strncpy(info->name, get_cpu_info().c_str(), MAX_STRING_LENGTH - 1);
+ info->total_memory_gb = get_mem_gb();
+ }
+}
+
+void list_available_accelerators() {
+ if (llamafile_has_gpu()) {
+ if (llamafile_has_metal()) {
+ fprintf(stderr, "Apple Metal\n");
+ } else if (llamafile_has_cuda()) {
+ int count = ggml_backend_cuda_get_device_count();
+ fprintf(stderr, "\n\033[0;32m==================== Available GPUs ====================\n\n");
+ for (int i = 0; i < count; i++) {
+ struct ggml_cuda_device_properties props;
+ ggml_backend_cuda_get_device_properties(i, &props);
+ fprintf(stderr, "%d: %s - %.2f GiB\n", i, props.name, props.totalGlobalMem / 1073741824.0);
+ }
+ } else {
+ fprintf(stderr, "No Accelerator support available\n");
+ }
+ } else {
+ fprintf(stderr, "No Accelerator support available\n");
+ }
+ fprintf(stderr, "\n======================================================================\n\033[0m");
+}
+
+void get_model_info(ModelInfo *info, llama_model *model) {
+ char buf[MAX_STRING_LENGTH];
+ llama_model_desc(model, buf, sizeof(buf));
+ strncpy(info->type, buf, sizeof(buf));
+ llama_model_meta_val_str(model, "general.name", buf, sizeof(buf));
+ strncpy(info->name, buf, sizeof(buf));
+
+ // check if the model name is empty, if it is then exit the program
+ if (strcmp(info->name, "") == 0) {
+ fprintf(stderr, "Error: Model name is empty. Please use a valid .gguf.\n");
+ exit(1);
+ }
+
+ llama_model_quant_str(model, buf, sizeof(buf));
+ strncpy(info->quant, buf, sizeof(buf));
+ llama_model_meta_val_str(model, "general.size_label", buf, sizeof(buf));
+ strncpy(info->size_label, buf, sizeof(buf));
+ info->size = llama_model_size(model);
+ info->params = llama_model_n_params(model);
+}
\ No newline at end of file
diff --git a/localscore/system.h b/localscore/system.h
new file mode 100644
index 00000000..fcad23bc
--- /dev/null
+++ b/localscore/system.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include <string>
+#include "llama.cpp/llama.h"
+
+#define MAX_STRING_LENGTH 256
+
+// Forward declaration if needed
+struct cmd_params;
+
+// Core data structures
+struct RuntimeInfo {
+ char llamafile_version[MAX_STRING_LENGTH];
+ char llama_commit[MAX_STRING_LENGTH];
+};
+
+struct SystemInfo {
+ char kernel_type[MAX_STRING_LENGTH];
+ char kernel_release[MAX_STRING_LENGTH];
+ char version[MAX_STRING_LENGTH];
+ char system_architecture[MAX_STRING_LENGTH];
+ char cpu[MAX_STRING_LENGTH];
+ double ram_gb;
+};
+
+struct AcceleratorInfo {
+ char name[MAX_STRING_LENGTH];
+ char manufacturer[MAX_STRING_LENGTH];
+ double total_memory_gb;
+ int core_count;
+ double capability;
+};
+
+struct ModelInfo {
+ char name[MAX_STRING_LENGTH];
+ char quant[MAX_STRING_LENGTH];
+ char size_label[MAX_STRING_LENGTH];
+ uint64_t size;
+ uint64_t params;
+ char type[MAX_STRING_LENGTH];
+};
+
+// Public interface
+void get_runtime_info(RuntimeInfo* info);
+void get_sys_info(SystemInfo* info);
+void get_accelerator_info(AcceleratorInfo* info, cmd_params* params);
+void list_available_accelerators();
+void get_model_info(ModelInfo* info, llama_model* model);
\ No newline at end of file
diff --git a/localscore/utils.h b/localscore/utils.h
new file mode 100644
index 00000000..f28b3de6
--- /dev/null
+++ b/localscore/utils.h
@@ -0,0 +1,191 @@
+#ifndef UTILS_H
+#define UTILS_H
+
+#include <string>
+#include <vector>
+#include <chrono>
+#include <sstream>
+#include <algorithm>
+#include <numeric>
+#include <cmath>
+#include <memory>
+#include <array>
+#include <cstdio>
+#include <stdexcept>
+#include <string.h>
+#include <stdarg.h>
+#include <math.h>
+
+namespace utils {
+
+ inline uint64_t get_time_ns() {
+ using clock = std::chrono::high_resolution_clock;
+ return std::chrono::nanoseconds(clock::now().time_since_epoch()).count();
+ }
+
+ template<class T>
+ inline std::string join(const std::vector<T>& values, const std::string& delim) {
+ std::ostringstream str;
+ for (size_t i = 0; i < values.size(); i++) {
+ str << values[i];
+ if (i < values.size() - 1) {
+ str << delim;
+ }
+ }
+ return str.str();
+ }
+
+ template<class T>
+ inline std::vector<T> split(const std::string& str, char delim) {
+ std::vector<T> values;
+ std::istringstream str_stream(str);
+ std::string token;
+ while (std::getline(str_stream, token, delim)) {
+ T value;
+ std::istringstream token_stream(token);
+ token_stream >> value;
+ values.push_back(value);
+ }
+ return values;
+ }
+
+ template<typename T, typename F>
+ inline std::vector<std::string> transform_to_str(const std::vector<T>& values, F f) {
+ std::vector<std::string> str_values;
+ std::transform(values.begin(), values.end(), std::back_inserter(str_values), f);
+ return str_values;
+ }
+
+ template<typename T>
+ inline T avg(const std::vector<T>& v) {
+ if (v.empty()) {
+ return 0;
+ }
+ T sum = std::accumulate(v.begin(), v.end(), T(0));
+ return sum / static_cast<T>(v.size());
+ }
+
+ template<typename T>
+ inline T stdev(const std::vector<T>& v) {
+ if (v.size() <= 1) {
+ return 0;
+ }
+ T mean = avg(v);
+ T sq_sum = std::inner_product(v.begin(), v.end(), v.begin(), T(0));
+ T stdev = std::sqrt(sq_sum / static_cast<T>(v.size() - 1) -
+ mean * mean * static_cast<T>(v.size()) / static_cast<T>(v.size() - 1));
+ return stdev;
+ }
+
+ inline std::string exec(const char* cmd) {
+ std::array<char, 128> buffer;
+ std::string result;
+ std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd, "r"), pclose);
+ if (!pipe) {
+ throw std::runtime_error("popen() failed!");
+ }
+ while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
+ result += buffer.data();
+ }
+ return result;
+ }
+
+ inline void print_centered(FILE *stream, int width, char decoration_char, const char *format, ...) {
+ char buffer[1024]; // Adjust size as needed
+
+ // Handle variable arguments
+ va_list args;
+ va_start(args, format);
+ vsnprintf(buffer, sizeof(buffer), format, args);
+ va_end(args);
+
+ const char *text = buffer;
+
+ // Calculate the visible length (excluding ANSI escape codes)
+ int visible_length = 0;
+ const char *ptr = text;
+ while (*ptr) {
+ if (*ptr == '\033') {
+ // Skip the escape sequence
+ ptr++;
+ if (*ptr == '[') {
+ ptr++;
+ while (*ptr && !isalpha(*ptr)) {
+ ptr++;
+ }
+ if (*ptr) ptr++; // Skip the final character of the sequence
+ continue;
+ }
+ }
+ visible_length++;
+ ptr++;
+ }
+
+ // Need at least 2 spaces to separate text from decorations
+ int remaining_width = width - visible_length - 2;
+
+ // Not enough space for decorations and spaces
+ if (remaining_width < 2) {
+ fprintf(stream, "%s\n", text);
+ return;
+ }
+
+ // Calculate how many decoration characters on each side
+ int decoration_count = remaining_width / 2;
+
+ // Calculate padding to truly center everything
+ int total_decorated_length = visible_length + 2 + (decoration_count * 2);
+ int padding = (width - total_decorated_length) / 2;
+
+ // Print left padding
+ for (int i = 0; i < padding; i++) {
+ fprintf(stream, " ");
+ }
+
+ // Print left decorations
+ for (int i = 0; i < decoration_count; i++) {
+ fprintf(stream, "%c", decoration_char);
+ }
+
+ // Print text with spaces
+ fprintf(stream, " %s ", text);
+
+ // Print right decorations
+ for (int i = 0; i < decoration_count; i++) {
+ fprintf(stream, "%c", decoration_char);
+ }
+
+ // Add extra padding on right if needed
+ int right_padding = width - total_decorated_length - padding;
+ for (int i = 0; i < right_padding; i++) {
+ fprintf(stream, " ");
+ }
+
+ fprintf(stream, "\n");
+ }
+
+ inline void sanitize_string(char *dest, const char *src, size_t max_len) {
+ size_t i = 0;
+ for (; i < max_len - 1 && src[i]; i++) {
+ // Only allow printable ASCII
+ dest[i] = (src[i] >= 32 && src[i] < 127) ? src[i] : ' ';
+ }
+ dest[i] = '\0';
+ }
+
+ /**
+ * Rounds a double value to the specified number of decimal places.
+ *
+ * @param value The value to round
+ * @param decimal_places The number of decimal places to round to
+ * @return The rounded value
+ */
+ inline double round_to_decimal(double value, int decimal_places) {
+ double multiplier = pow(10.0, decimal_places);
+ return round(value * multiplier) / multiplier;
+ }
+
+
+} // namespace utils
+
+#endif // UTILS_H