Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make the build go a little faster
Justine Tunney8b9be968d3d2
diff --git a/build/rules.mk b/build/rules.mkindex b5d4964f..e31146c8 100644--- a/build/rules.mk+++ b/build/rules.mk@@ -16,6 +16,10 @@ o/$(MODE)/%.o: %.cc $(COSMOCC)@mkdir -p $(@D)$(COMPILE.cc) -o $@ $<+o/$(MODE)/%.o: %.cc $(COSMOCC)+ @mkdir -p $(@D)+ $(COMPILE.cc) -o $@ $<+o/$(MODE)/%.o: %.cpp $(COSMOCC)@mkdir -p $(@D)$(COMPILE.cc) -o $@ $<@@ -41,8 +45,5 @@ o/$(MODE)/%.zip.o: % $(COSMOCC)$(ZIPOBJ) $(ZIPOBJ_FLAGS) -a aarch64 -o $(dir $@)/.aarch64/$(notdir $@) $<$(PREFIX)/bin/ape: $(COSMOCC) # cosmocc toolchain setup in restricted ci context- # Install ape loader$(INSTALL) $(COSMOCC)/bin/ape-$(ARCH).elf $(PREFIX)/bin/ape-- # Config binfmt_misc to use ape loader for ape.elf files- echo ':APE:M::MZqFpD::/usr/bin/ape:' > /proc/sys/fs/binfmt_misc/register\ No newline at end of file+ echo ':APE:M::MZqFpD::/usr/bin/ape:' > /proc/sys/fs/binfmt_misc/registerdiff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mkindex a37eb07d..78b05d31 100644--- a/llama.cpp/BUILD.mk+++ b/llama.cpp/BUILD.mk@@ -32,6 +32,10 @@ $(LLAMA_CPP_OBJS): private \-DGGML_MULTIPLATFORM \-DGGML_USE_LLAMAFILE+o/$(MODE)/llama.cpp/common.o \+o/$(MODE)/llama.cpp/llama.o: private \+ CCFLAGS += -O+o/$(MODE)/llama.cpp/ggml-alloc.o \o/$(MODE)/llama.cpp/ggml-backend.o \o/$(MODE)/llama.cpp/grammar-parser.o \diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex e1313a8b..d92dd962 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -76,112 +76,6 @@using json = nlohmann::ordered_json;-int32_t get_num_physical_cores() {- if (IsLinux()) {- // enumerate the set of thread siblings, num entries is num cores- std::unordered_set<std::string> siblings;- for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {- std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"- + std::to_string(cpu) + "/topology/thread_siblings");- if (!thread_siblings.is_open()) {- break; // no more cpus- }- std::string line;- if (std::getline(thread_siblings, line)) {- siblings.insert(line);- }- }- if (!siblings.empty()) {- return static_cast<int32_t>(siblings.size());- }- }- int32_t num_physical_cores;- size_t len = sizeof(num_physical_cores);- int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);- if (result == 0) {- return num_physical_cores;- }- result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);- if (result == 0) {- return num_physical_cores;- }- unsigned int n_threads = std::thread::hardware_concurrency();- return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;-}--#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)-#include <pthread.h>--static void cpuid(unsigned leaf, unsigned subleaf,- unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {- __asm__("movq\t%%rbx,%%rsi\n\t"- "cpuid\n\t"- "xchgq\t%%rbx,%%rsi"- : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)- : "0"(leaf), "2"(subleaf));-}--static int pin_cpu(int cpu) {- cpu_set_t mask;- CPU_ZERO(&mask);- CPU_SET(cpu, &mask);- return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);-}--static bool is_hybrid_cpu(void) {- unsigned eax, ebx, ecx, edx;- cpuid(7, 0, &eax, &ebx, &ecx, &edx);- return !!(edx & (1u << 15));-}--static bool is_running_on_efficiency_core(void) {- unsigned eax, ebx, ecx, edx;- cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);- int intel_atom = 0x20;- int core_type = (eax & 0xff000000u) >> 24;- return core_type == intel_atom;-}--static int count_math_cpus(int cpu_count) {- int result = 0;- for (int cpu = 0; cpu < cpu_count; ++cpu) {- if (pin_cpu(cpu)) {- return -1;- }- if (is_running_on_efficiency_core()) {- continue; // efficiency cores harm lockstep threading- }- ++cpu; // hyperthreading isn't useful for linear algebra- ++result;- }- return result;-}--#endif // __x86_64__ && __linux__--/**- * Returns number of CPUs on system that are useful for math.- */-int cpu_get_num_math() {-#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)- int cpu_count = sysconf(_SC_NPROCESSORS_ONLN);- if (cpu_count < 1) {- return get_num_physical_cores();- }- if (is_hybrid_cpu()) {- cpu_set_t affinity;- if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {- int result = count_math_cpus(cpu_count);- pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);- if (result > 0) {- return result;- }- }- }-#endif- return get_num_physical_cores();-}-void process_escapes(std::string & input) {std::size_t input_len = input.length();std::size_t output_idx = 0;diff --git a/llama.cpp/cores.cpp b/llama.cpp/cores.cppnew file mode 100644index 00000000..f138ea0f--- /dev/null+++ b/llama.cpp/cores.cpp@@ -0,0 +1,111 @@+#include <cosmo.h>+#include <fstream>+#include <thread>+#include <unistd.h>+#include <unordered_set>++int32_t get_num_physical_cores() {+ if (IsLinux()) {+ // enumerate the set of thread siblings, num entries is num cores+ std::unordered_set<std::string> siblings;+ for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {+ std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"+ + std::to_string(cpu) + "/topology/thread_siblings");+ if (!thread_siblings.is_open()) {+ break; // no more cpus+ }+ std::string line;+ if (std::getline(thread_siblings, line)) {+ siblings.insert(line);+ }+ }+ if (!siblings.empty()) {+ return static_cast<int32_t>(siblings.size());+ }+ }+ int32_t num_physical_cores;+ size_t len = sizeof(num_physical_cores);+ int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);+ if (result == 0) {+ return num_physical_cores;+ }+ result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);+ if (result == 0) {+ return num_physical_cores;+ }+ unsigned int n_threads = std::thread::hardware_concurrency();+ return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;+}++#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)+#include <pthread.h>++static void cpuid(unsigned leaf, unsigned subleaf,+ unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {+ __asm__("movq\t%%rbx,%%rsi\n\t"+ "cpuid\n\t"+ "xchgq\t%%rbx,%%rsi"+ : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)+ : "0"(leaf), "2"(subleaf));+}++static int pin_cpu(int cpu) {+ cpu_set_t mask;+ CPU_ZERO(&mask);+ CPU_SET(cpu, &mask);+ return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);+}++static bool is_hybrid_cpu(void) {+ unsigned eax, ebx, ecx, edx;+ cpuid(7, 0, &eax, &ebx, &ecx, &edx);+ return !!(edx & (1u << 15));+}++static bool is_running_on_efficiency_core(void) {+ unsigned eax, ebx, ecx, edx;+ cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);+ int intel_atom = 0x20;+ int core_type = (eax & 0xff000000u) >> 24;+ return core_type == intel_atom;+}++static int count_math_cpus(int cpu_count) {+ int result = 0;+ for (int cpu = 0; cpu < cpu_count; ++cpu) {+ if (pin_cpu(cpu)) {+ return -1;+ }+ if (is_running_on_efficiency_core()) {+ continue; // efficiency cores harm lockstep threading+ }+ ++cpu; // hyperthreading isn't useful for linear algebra+ ++result;+ }+ return result;+}++#endif // __x86_64__ && __linux__++/**+ * Returns number of CPUs on system that are useful for math.+ */+int cpu_get_num_math() {+#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)+ int cpu_count = sysconf(_SC_NPROCESSORS_ONLN);+ if (cpu_count < 1) {+ return get_num_physical_cores();+ }+ if (is_hybrid_cpu()) {+ cpu_set_t affinity;+ if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {+ int result = count_math_cpus(cpu_count);+ pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);+ if (result > 0) {+ return result;+ }+ }+ }+#endif+ return get_num_physical_cores();+}diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 80e1625d..49362443 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -18975,6 +18975,8 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {}#endif+ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, 0);+#ifdef LLAMAFILE_SYNC_REPORTg_sync.stamp = rdtsc();unsigned long old = 0;diff --git a/llama.cpp/llama-bench/llama-bench.cpp b/llama.cpp/llama-bench/llama-bench.cppindex 83fd3479..4f919ac3 100644--- a/llama.cpp/llama-bench/llama-bench.cpp+++ b/llama.cpp/llama-bench/llama-bench.cpp@@ -265,7 +265,7 @@ struct cmd_params {};static const cmd_params cmd_params_defaults = {- /* model */ {"models/7B/ggml-model-q4_0.gguf"},+ /* model */ {}, // [jart] no default guessing/* n_prompt */ {512},/* n_gen */ {16},/* n_pg */ {},@@ -359,7 +359,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {for (int i = 1; i < argc; i++) {arg = argv[i];if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {- std::replace(arg.begin(), arg.end(), '_', '-');+ std::replace (arg.begin(), arg.end(), '_', '-');}if (arg == "-h" || arg == "--help") {@@ -581,8 +581,11 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {}}if (invalid_param) {- fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());- print_usage(argc, argv);+ fprintf(stderr, "%s: invalid parameter for argument: %s\n", program_invocation_name, arg.c_str());+ exit(1);+ }+ if (params.model.empty()) {+ fprintf(stderr, "%s: missing operand\n", program_invocation_name, arg.c_str());exit(1);}diff --git a/llama.cpp/unicode.cpp b/llama.cpp/unicode.cppindex 056a4c74..59710584 100644--- a/llama.cpp/unicode.cpp+++ b/llama.cpp/unicode.cpp@@ -2,6 +2,7 @@#include "unicode-data.h"#include <cassert>+#include <algorithm> // [jart]#include <cstddef>#include <cstdint>#include <map>diff --git a/stable-diffusion.cpp/BUILD.mk b/stable-diffusion.cpp/BUILD.mkindex 60cfce1c..2483214b 100644--- a/stable-diffusion.cpp/BUILD.mk+++ b/stable-diffusion.cpp/BUILD.mk@@ -14,8 +14,7 @@ STABLE_DIFFUSION_CPP_SRCS = $(STABLE_DIFFUSION_CPP_SRCS_C) $(STABLE_DIFFUSION_CPSTABLE_DIFFUSION_CPP_OBJS = \$(LLAMAFILE_OBJS) \$(STABLE_DIFFUSION_CPP_SRCS_C:%.c=o/$(MODE)/%.o) \- $(STABLE_DIFFUSION_CPP_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \- $(STABLE_DIFFUSION_CPP_FILES:%=o/$(MODE)/%.zip.o)+ $(STABLE_DIFFUSION_CPP_SRCS_CPP:%.cpp=o/$(MODE)/%.o)o/$(MODE)/stable-diffusion.cpp/stable-diffusion.cpp.a: $(STABLE_DIFFUSION_CPP_OBJS)@@ -34,6 +33,7 @@ o/$(MODE)/stable-diffusion.cpp/main: \o/$(MODE)/stb/stb.a$(STABLE_DIFFUSION_CPP_OBJS): stable-diffusion.cpp/BUILD.mk+$(STABLE_DIFFUSION_CPP_OBJS): private CFLAGS += -O.PHONY: o/$(MODE)/stable-diffusion.cppo/$(MODE)/stable-diffusion.cpp: \diff --git a/stable-diffusion.cpp/main.cpp b/stable-diffusion.cpp/main.cppindex 4f5ed3a8..4f3b126d 100644--- a/stable-diffusion.cpp/main.cpp+++ b/stable-diffusion.cpp/main.cpp@@ -11,6 +11,8 @@#include "stb/stb_image.h"#include "stb/stb_image_write.h"#include "stb/stb_image_resize.h"+#include "llamafile/llamafile.h"+#include "llamafile/debug.h"const char* rng_type_to_str[] = {"std_default",@@ -198,7 +200,33 @@ void parse_args(int argc, const char** argv, SDParams& params) {for (int i = 1; i < argc; i++) {arg = argv[i];- if (arg == "-t" || arg == "--threads") {+ // [jart]+ if (arg == "--fast") {+ FLAG_precise = false;+ } else if (arg == "--precise") {+ FLAG_precise = true;+ } else if (arg == "--trap") {+ FLAG_trap = true;+ FLAG_unsecure = true; // for better backtraces+ llamafile_trapping_enabled(+1);+ } else if (arg == "--unsecure") {+ FLAG_unsecure = true;+ } else if (arg == "--nocompile") {+ FLAG_nocompile = true;+ } else if (arg == "--recompile") {+ FLAG_recompile = true;+ } else if (arg == "--tinyblas") {+ FLAG_tinyblas = true; // undocumented+ } else if (arg == "--gpu") {+ if (++i >= argc) {+ invalid_arg = true;+ }+ FLAG_gpu = llamafile_gpu_parse(argv[i]);+ if (FLAG_gpu == LLAMAFILE_GPU_ERROR) {+ fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);+ exit(1);+ }+ } else if (arg == "-t" || arg == "--threads") {if (++i >= argc) {invalid_arg = true;break;diff --git a/stb/README.llamafile b/stb/README.llamafilenew file mode 100644index 00000000..74710f1a--- /dev/null+++ b/stb/README.llamafile@@ -0,0 +1,15 @@+DESCRIPTION++ Single-file public domain libraries for C/C++++LICENSE++ Public Domain or MIT++ORIGIN++ https://github.com/nothings/stb++LOCAL MODIFICATIONS++ - Add "e" to fopen() calls for O_CLOEXEC behavior