Skip to content

Distribute and run LLMs with a single file.

Make the build go a little faster

Justine Tunney8b9be968d3d2
diff --git a/build/rules.mk b/build/rules.mk
index b5d4964f..e31146c8 100644
--- a/build/rules.mk
+++ b/build/rules.mk
@@ -16,6 +16,10 @@ o/$(MODE)/%.o: %.cc $(COSMOCC)
@mkdir -p $(@D)
$(COMPILE.cc) -o $@ $<
+o/$(MODE)/%.o: %.cc $(COSMOCC)
+ @mkdir -p $(@D)
+ $(COMPILE.cc) -o $@ $<
+
o/$(MODE)/%.o: %.cpp $(COSMOCC)
@mkdir -p $(@D)
$(COMPILE.cc) -o $@ $<
@@ -41,8 +45,5 @@ o/$(MODE)/%.zip.o: % $(COSMOCC)
$(ZIPOBJ) $(ZIPOBJ_FLAGS) -a aarch64 -o $(dir $@)/.aarch64/$(notdir $@) $<
$(PREFIX)/bin/ape: $(COSMOCC) # cosmocc toolchain setup in restricted ci context
- # Install ape loader
$(INSTALL) $(COSMOCC)/bin/ape-$(ARCH).elf $(PREFIX)/bin/ape
-
- # Config binfmt_misc to use ape loader for ape.elf files
- echo ':APE:M::MZqFpD::/usr/bin/ape:' > /proc/sys/fs/binfmt_misc/register
\ No newline at end of file
+ echo ':APE:M::MZqFpD::/usr/bin/ape:' > /proc/sys/fs/binfmt_misc/register
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
index a37eb07d..78b05d31 100644
--- a/llama.cpp/BUILD.mk
+++ b/llama.cpp/BUILD.mk
@@ -32,6 +32,10 @@ $(LLAMA_CPP_OBJS): private \
-DGGML_MULTIPLATFORM \
-DGGML_USE_LLAMAFILE
+o/$(MODE)/llama.cpp/common.o \
+o/$(MODE)/llama.cpp/llama.o: private \
+ CCFLAGS += -O
+
o/$(MODE)/llama.cpp/ggml-alloc.o \
o/$(MODE)/llama.cpp/ggml-backend.o \
o/$(MODE)/llama.cpp/grammar-parser.o \
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index e1313a8b..d92dd962 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -76,112 +76,6 @@
using json = nlohmann::ordered_json;
-int32_t get_num_physical_cores() {
- if (IsLinux()) {
- // enumerate the set of thread siblings, num entries is num cores
- std::unordered_set<std::string> siblings;
- for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {
- std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"
- + std::to_string(cpu) + "/topology/thread_siblings");
- if (!thread_siblings.is_open()) {
- break; // no more cpus
- }
- std::string line;
- if (std::getline(thread_siblings, line)) {
- siblings.insert(line);
- }
- }
- if (!siblings.empty()) {
- return static_cast<int32_t>(siblings.size());
- }
- }
- int32_t num_physical_cores;
- size_t len = sizeof(num_physical_cores);
- int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);
- if (result == 0) {
- return num_physical_cores;
- }
- result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);
- if (result == 0) {
- return num_physical_cores;
- }
- unsigned int n_threads = std::thread::hardware_concurrency();
- return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;
-}
-
-#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)
-#include <pthread.h>
-
-static void cpuid(unsigned leaf, unsigned subleaf,
- unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {
- __asm__("movq\t%%rbx,%%rsi\n\t"
- "cpuid\n\t"
- "xchgq\t%%rbx,%%rsi"
- : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)
- : "0"(leaf), "2"(subleaf));
-}
-
-static int pin_cpu(int cpu) {
- cpu_set_t mask;
- CPU_ZERO(&mask);
- CPU_SET(cpu, &mask);
- return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);
-}
-
-static bool is_hybrid_cpu(void) {
- unsigned eax, ebx, ecx, edx;
- cpuid(7, 0, &eax, &ebx, &ecx, &edx);
- return !!(edx & (1u << 15));
-}
-
-static bool is_running_on_efficiency_core(void) {
- unsigned eax, ebx, ecx, edx;
- cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);
- int intel_atom = 0x20;
- int core_type = (eax & 0xff000000u) >> 24;
- return core_type == intel_atom;
-}
-
-static int count_math_cpus(int cpu_count) {
- int result = 0;
- for (int cpu = 0; cpu < cpu_count; ++cpu) {
- if (pin_cpu(cpu)) {
- return -1;
- }
- if (is_running_on_efficiency_core()) {
- continue; // efficiency cores harm lockstep threading
- }
- ++cpu; // hyperthreading isn't useful for linear algebra
- ++result;
- }
- return result;
-}
-
-#endif // __x86_64__ && __linux__
-
-/**
- * Returns number of CPUs on system that are useful for math.
- */
-int cpu_get_num_math() {
-#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)
- int cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
- if (cpu_count < 1) {
- return get_num_physical_cores();
- }
- if (is_hybrid_cpu()) {
- cpu_set_t affinity;
- if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {
- int result = count_math_cpus(cpu_count);
- pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);
- if (result > 0) {
- return result;
- }
- }
- }
-#endif
- return get_num_physical_cores();
-}
-
void process_escapes(std::string & input) {
std::size_t input_len = input.length();
std::size_t output_idx = 0;
diff --git a/llama.cpp/cores.cpp b/llama.cpp/cores.cpp
new file mode 100644
index 00000000..f138ea0f
--- /dev/null
+++ b/llama.cpp/cores.cpp
@@ -0,0 +1,111 @@
+#include <cosmo.h>
+#include <fstream>
+#include <thread>
+#include <unistd.h>
+#include <unordered_set>
+
+int32_t get_num_physical_cores() {
+ if (IsLinux()) {
+ // enumerate the set of thread siblings, num entries is num cores
+ std::unordered_set<std::string> siblings;
+ for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {
+ std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"
+ + std::to_string(cpu) + "/topology/thread_siblings");
+ if (!thread_siblings.is_open()) {
+ break; // no more cpus
+ }
+ std::string line;
+ if (std::getline(thread_siblings, line)) {
+ siblings.insert(line);
+ }
+ }
+ if (!siblings.empty()) {
+ return static_cast<int32_t>(siblings.size());
+ }
+ }
+ int32_t num_physical_cores;
+ size_t len = sizeof(num_physical_cores);
+ int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);
+ if (result == 0) {
+ return num_physical_cores;
+ }
+ result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);
+ if (result == 0) {
+ return num_physical_cores;
+ }
+ unsigned int n_threads = std::thread::hardware_concurrency();
+ return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;
+}
+
+#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)
+#include <pthread.h>
+
+static void cpuid(unsigned leaf, unsigned subleaf,
+ unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {
+ __asm__("movq\t%%rbx,%%rsi\n\t"
+ "cpuid\n\t"
+ "xchgq\t%%rbx,%%rsi"
+ : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)
+ : "0"(leaf), "2"(subleaf));
+}
+
+static int pin_cpu(int cpu) {
+ cpu_set_t mask;
+ CPU_ZERO(&mask);
+ CPU_SET(cpu, &mask);
+ return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);
+}
+
+static bool is_hybrid_cpu(void) {
+ unsigned eax, ebx, ecx, edx;
+ cpuid(7, 0, &eax, &ebx, &ecx, &edx);
+ return !!(edx & (1u << 15));
+}
+
+static bool is_running_on_efficiency_core(void) {
+ unsigned eax, ebx, ecx, edx;
+ cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);
+ int intel_atom = 0x20;
+ int core_type = (eax & 0xff000000u) >> 24;
+ return core_type == intel_atom;
+}
+
+static int count_math_cpus(int cpu_count) {
+ int result = 0;
+ for (int cpu = 0; cpu < cpu_count; ++cpu) {
+ if (pin_cpu(cpu)) {
+ return -1;
+ }
+ if (is_running_on_efficiency_core()) {
+ continue; // efficiency cores harm lockstep threading
+ }
+ ++cpu; // hyperthreading isn't useful for linear algebra
+ ++result;
+ }
+ return result;
+}
+
+#endif // __x86_64__ && __linux__
+
+/**
+ * Returns number of CPUs on system that are useful for math.
+ */
+int cpu_get_num_math() {
+#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)
+ int cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
+ if (cpu_count < 1) {
+ return get_num_physical_cores();
+ }
+ if (is_hybrid_cpu()) {
+ cpu_set_t affinity;
+ if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {
+ int result = count_math_cpus(cpu_count);
+ pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);
+ if (result > 0) {
+ return result;
+ }
+ }
+ }
+#endif
+ return get_num_physical_cores();
+}
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 80e1625d..49362443 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -18975,6 +18975,8 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
}
#endif
+ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, 0);
+
#ifdef LLAMAFILE_SYNC_REPORT
g_sync.stamp = rdtsc();
unsigned long old = 0;
diff --git a/llama.cpp/llama-bench/llama-bench.cpp b/llama.cpp/llama-bench/llama-bench.cpp
index 83fd3479..4f919ac3 100644
--- a/llama.cpp/llama-bench/llama-bench.cpp
+++ b/llama.cpp/llama-bench/llama-bench.cpp
@@ -265,7 +265,7 @@ struct cmd_params {
};
static const cmd_params cmd_params_defaults = {
- /* model */ {"models/7B/ggml-model-q4_0.gguf"},
+ /* model */ {}, // [jart] no default guessing
/* n_prompt */ {512},
/* n_gen */ {16},
/* n_pg */ {},
@@ -359,7 +359,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
for (int i = 1; i < argc; i++) {
arg = argv[i];
if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {
- std::replace(arg.begin(), arg.end(), '_', '-');
+ std::replace (arg.begin(), arg.end(), '_', '-');
}
if (arg == "-h" || arg == "--help") {
@@ -581,8 +581,11 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
}
}
if (invalid_param) {
- fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
- print_usage(argc, argv);
+ fprintf(stderr, "%s: invalid parameter for argument: %s\n", program_invocation_name, arg.c_str());
+ exit(1);
+ }
+ if (params.model.empty()) {
+ fprintf(stderr, "%s: missing operand\n", program_invocation_name, arg.c_str());
exit(1);
}
diff --git a/llama.cpp/unicode.cpp b/llama.cpp/unicode.cpp
index 056a4c74..59710584 100644
--- a/llama.cpp/unicode.cpp
+++ b/llama.cpp/unicode.cpp
@@ -2,6 +2,7 @@
#include "unicode-data.h"
#include <cassert>
+#include <algorithm> // [jart]
#include <cstddef>
#include <cstdint>
#include <map>
diff --git a/stable-diffusion.cpp/BUILD.mk b/stable-diffusion.cpp/BUILD.mk
index 60cfce1c..2483214b 100644
--- a/stable-diffusion.cpp/BUILD.mk
+++ b/stable-diffusion.cpp/BUILD.mk
@@ -14,8 +14,7 @@ STABLE_DIFFUSION_CPP_SRCS = $(STABLE_DIFFUSION_CPP_SRCS_C) $(STABLE_DIFFUSION_CP
STABLE_DIFFUSION_CPP_OBJS = \
$(LLAMAFILE_OBJS) \
$(STABLE_DIFFUSION_CPP_SRCS_C:%.c=o/$(MODE)/%.o) \
- $(STABLE_DIFFUSION_CPP_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \
- $(STABLE_DIFFUSION_CPP_FILES:%=o/$(MODE)/%.zip.o)
+ $(STABLE_DIFFUSION_CPP_SRCS_CPP:%.cpp=o/$(MODE)/%.o)
o/$(MODE)/stable-diffusion.cpp/stable-diffusion.cpp.a: $(STABLE_DIFFUSION_CPP_OBJS)
@@ -34,6 +33,7 @@ o/$(MODE)/stable-diffusion.cpp/main: \
o/$(MODE)/stb/stb.a
$(STABLE_DIFFUSION_CPP_OBJS): stable-diffusion.cpp/BUILD.mk
+$(STABLE_DIFFUSION_CPP_OBJS): private CFLAGS += -O
.PHONY: o/$(MODE)/stable-diffusion.cpp
o/$(MODE)/stable-diffusion.cpp: \
diff --git a/stable-diffusion.cpp/main.cpp b/stable-diffusion.cpp/main.cpp
index 4f5ed3a8..4f3b126d 100644
--- a/stable-diffusion.cpp/main.cpp
+++ b/stable-diffusion.cpp/main.cpp
@@ -11,6 +11,8 @@
#include "stb/stb_image.h"
#include "stb/stb_image_write.h"
#include "stb/stb_image_resize.h"
+#include "llamafile/llamafile.h"
+#include "llamafile/debug.h"
const char* rng_type_to_str[] = {
"std_default",
@@ -198,7 +200,33 @@ void parse_args(int argc, const char** argv, SDParams& params) {
for (int i = 1; i < argc; i++) {
arg = argv[i];
- if (arg == "-t" || arg == "--threads") {
+ // [jart]
+ if (arg == "--fast") {
+ FLAG_precise = false;
+ } else if (arg == "--precise") {
+ FLAG_precise = true;
+ } else if (arg == "--trap") {
+ FLAG_trap = true;
+ FLAG_unsecure = true; // for better backtraces
+ llamafile_trapping_enabled(+1);
+ } else if (arg == "--unsecure") {
+ FLAG_unsecure = true;
+ } else if (arg == "--nocompile") {
+ FLAG_nocompile = true;
+ } else if (arg == "--recompile") {
+ FLAG_recompile = true;
+ } else if (arg == "--tinyblas") {
+ FLAG_tinyblas = true; // undocumented
+ } else if (arg == "--gpu") {
+ if (++i >= argc) {
+ invalid_arg = true;
+ }
+ FLAG_gpu = llamafile_gpu_parse(argv[i]);
+ if (FLAG_gpu == LLAMAFILE_GPU_ERROR) {
+ fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);
+ exit(1);
+ }
+ } else if (arg == "-t" || arg == "--threads") {
if (++i >= argc) {
invalid_arg = true;
break;
diff --git a/stb/README.llamafile b/stb/README.llamafile
new file mode 100644
index 00000000..74710f1a
--- /dev/null
+++ b/stb/README.llamafile
@@ -0,0 +1,15 @@
+DESCRIPTION
+
+ Single-file public domain libraries for C/C++
+
+LICENSE
+
+ Public Domain or MIT
+
+ORIGIN
+
+ https://github.com/nothings/stb
+
+LOCAL MODIFICATIONS
+
+ - Add "e" to fopen() calls for O_CLOEXEC behavior