Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Synchronize after every op
Justine Tunney9cf736355383
diff --git a/build/config.mk b/build/config.mkindex c2973a1b..35acbf72 100644--- a/build/config.mk+++ b/build/config.mk@@ -13,8 +13,8 @@ MKDEPS = $(COSMOCC)/bin/mkdepsINSTALL = installARFLAGS = rcsD-CCFLAGS = -g -O3 -fexceptions-CPPFLAGS_ = -iquote. -mcosmo -DGGML_MULTIPLATFORM -Wno-attributes+CCFLAGS = -g -O3 -fexceptions -fsignaling-nans+CPPFLAGS_ = -iquote. -mcosmo -DGGML_MULTIPLATFORM -Wno-attributes -DLLAMAFILE_DEBUGTARGET_ARCH = -Xx86_64-mavx -Xx86_64-mtune=znver4TMPDIR = o//tmpdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex f46ef6ae..0a777802 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -233,6 +233,12 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paif (arg == "--cli") {return true;}+ if (arg == "--trap") {+ FLAG_trap = true;+ FLAG_unsecure = true; // for better backtraces+ llamafile_trapping_enabled(+1);+ return true;+ }if (arg == "--unsecure") {FLAG_unsecure = true;return true;diff --git a/llama.cpp/console.cpp b/llama.cpp/console.cppindex 999f8045..144f69f1 100644--- a/llama.cpp/console.cpp+++ b/llama.cpp/console.cpp@@ -1,13 +1,15 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "console.h"+#include <vector>#include <iostream>-#include <climits>#include <sys/ioctl.h>#include <unistd.h>#include <wchar.h>+#include <cosmo.h>#include <stdio.h>#include <stdlib.h>#include <signal.h>@@ -30,6 +32,7 @@ namespace console {static bool advanced_display = false;static bool simple_io = true;+static bool should_close_tty = false;static display_t current_display = reset;static FILE* out = stdout;static FILE* tty = nullptr;@@ -40,19 +43,32 @@ static termios initial_state;//void init(bool use_simple_io, bool use_advanced_display) {- advanced_display = use_advanced_display;+ should_close_tty = false;simple_io = use_simple_io;+ advanced_display = use_advanced_display;if (!simple_io) {- struct termios new_termios;- tcgetattr(STDIN_FILENO, &initial_state);- new_termios = initial_state;- new_termios.c_lflag &= ~(ICANON | ECHO);- new_termios.c_cc[VMIN] = 1;- new_termios.c_cc[VTIME] = 0;- tcsetattr(STDIN_FILENO, TCSANOW, &new_termios);tty = fopen("/dev/tty", "w+e");+ if (tty) {+ should_close_tty = true;+ } else if (IsLinux() || IsOpenbsd()) {+ // this could happen because pledge() blocked us+ tty = fdopen(0, "w+e");+ }if (tty != nullptr) {- out = tty;+ if (!tcgetattr(fileno(tty), &initial_state)) {+ out = tty;+ struct termios new_termios = initial_state;+ new_termios.c_lflag &= ~(ICANON | ECHO);+ new_termios.c_cc[VMIN] = 1;+ new_termios.c_cc[VTIME] = 0;+ tcsetattr(fileno(tty), TCSANOW, &new_termios);+ } else {+ simple_io = true;+ fclose(tty);+ tty = 0;+ }+ } else {+ simple_io = true;}}setlocale(LC_ALL, "");@@ -64,11 +80,14 @@ void cleanup() {// Restore settingsif (!simple_io) {if (tty != nullptr) {- out = stdout;- fclose(tty);+ fflush(tty);+ tcsetattr(fileno(tty), TCSANOW, &initial_state);+ if (should_close_tty) {+ fclose(tty);+ }tty = nullptr;+ out = stdout;}- tcsetattr(STDIN_FILENO, TCSANOW, &initial_state);}}diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 194ecca6..510cca01 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -117,9 +117,9 @@ void ggml_print_backtrace(void) {/*#define GGML_PERF*/#define GGML_DEBUG 0-#define GGML_GELU_FP16-#define GGML_GELU_QUICK_FP16-#define GGML_SILU_FP16+// #define GGML_GELU_FP16+// #define GGML_GELU_QUICK_FP16+// #define GGML_SILU_FP16// #define GGML_CROSS_ENTROPY_EXP_FP16// #define GGML_FLASH_ATTN_EXP_FP16@@ -2679,6 +2679,7 @@ static inline int ggml_up(int n, int m) {struct ggml_context * ggml_init(struct ggml_init_params params) {// make this function thread safeggml_critical_section_start();+ llamafile_trapping_enabled(-1);static bool is_first_call = true;@@ -2750,6 +2751,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {if (ctx == NULL) {GGML_PRINT_DEBUG("%s: no unused context found\n", __func__);+ llamafile_trapping_enabled(+1);ggml_critical_section_end();return NULL;@@ -2781,6 +2783,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {GGML_PRINT_DEBUG("%s: context initialized\n", __func__);+ llamafile_trapping_enabled(+1);ggml_critical_section_end();return ctx;@@ -7105,19 +7108,12 @@ void ggml_syncthreads(struct ggml_compute_params * params) {// ops must call this before accessing wdata// otherwise overlapping threads on previous op might clobbervoid *ggml_acquire_wdata(struct ggml_compute_params * params) {- if (params->limbo) {- ggml_syncthreads(params);- }- params->limbo = true;return params->wdata;}// ops should call this after writing to wdata before readingvoid ggml_release_wdata(struct ggml_compute_params * params) {- if (params->limbo) {- ggml_syncthreads(params);- params->limbo = false;- }+ ggml_syncthreads(params);}////////////////////////////////////////////////////////////////////////////////@@ -11531,12 +11527,9 @@ static void ggml_compute_forward_set_f32(if (!inplace) {if (!params->ith) {- // memcpy needs to be synchronized across threads to avoid race conditions.- // => do it in INIT phase- memcpy(- ((char *) dst->data),- ((char *) src0->data),- ggml_nbytes(dst));+ memcpy(((char *) dst->data),+ ((char *) src0->data),+ ggml_nbytes(dst));}ggml_syncthreads(params);}@@ -17759,35 +17752,6 @@ struct ggml_compute_state {enum ggml_status ec;};-// returns true if `src` is direct dependency of `node`-static bool ggml_has_src(const struct ggml_tensor * node,- const struct ggml_tensor * src) {- for (int i = 0; i < GGML_MAX_SRC; ++i) {- if (node->src[i] == src) {- return true;- }- }- return false;-}--// returns true if `dest` depends on any outputs since `mark`-//-// - cgraph->nodes[dest] is about to be executed-// - syncthreads() last happened right before cgraph->nodes[mark] executed-// - syncthreads() has 1.8 µs overhead minimum on Apple M2 when nth == 12-//-static bool ggml_needs_barrier(const struct ggml_cgraph * cgraph, int mark, int dest) {- assert(mark >= 0);- assert(mark <= dest);- assert(dest < cgraph->n_nodes);- for (; mark < dest; ++mark) {- if (ggml_has_src(cgraph->nodes[dest], cgraph->nodes[mark])) {- return true;- }- }- return false;-}-static thread_ret_t ggml_graph_compute_thread(void * data) {struct ggml_compute_state * state = (struct ggml_compute_state *) data;const struct ggml_cgraph * cgraph = state->shared->cgraph;@@ -17799,11 +17763,16 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {/*.wsize =*/ cplan->work_size,/*.wdata =*/ cplan->work_data,/*.barrier =*/ &state->shared->barrier,- /*.limbo =*/ false,};set_numa_thread_affinity(state->ith);+#ifdef LLAMAFILE_DEBUG+ if (FLAG_trap) {+ llamafile_trapping_enabled(+1);+ }+#endif+#ifdef GGML_PERFint64_t start_cycles, start_time_us;if (!state->ith) {@@ -17819,17 +17788,19 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {return 0;}- if (ggml_needs_barrier(cgraph, mark, node_n)) {- ggml_syncthreads(¶ms);- mark = node_n;- }+#ifdef LLAMAFILE_DEBUG+ llamafile_debug_op_index = node_n;+#endifstruct ggml_tensor *node = cgraph->nodes[node_n];params.nth = state->shared->n_threads;ggml_compute_forward(¶ms, node);+ // this barrier could potentially be eliminated in 15%+ of cases+ // however, it would give rise to ghoulish errors w/ little gain+ ggml_syncthreads(¶ms);+#if GGML_PERF- ggml_syncthreads(&state->shared->barrier);if (!state->ith) {int64_t end_cycles = ggml_perf_cycles();int64_t end_time_us = ggml_perf_time_us();@@ -17844,8 +17815,6 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {#endif}- ggml_syncthreads(¶ms);-return 0;}@@ -18075,6 +18044,10 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl};struct ggml_compute_state * workers = alloca(sizeof(struct ggml_compute_state)*n_threads);+#ifdef LLAMAFILE_DEBUG+ llamafile_debug_graph = cgraph;+#endif+// create thread poolif (n_threads > 1) {for (int j = 1; j < n_threads; ++j) {@@ -18134,6 +18107,10 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl// fprintf(stderr, "%6d barriers %6d ops\n", state_shared.barrier.phase, cgraph->n_nodes);+#ifdef LLAMAFILE_DEBUG+ llamafile_debug_graph = 0;+#endif+return compute_status;}diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex eb39dbc7..ff93b62f 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -678,7 +678,6 @@ extern "C" {void * wdata;struct ggml_barrier * barrier;- bool limbo;};GGML_API void ggml_syncthreads (struct ggml_compute_params *);diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 2d641f76..84388b1e 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -184,20 +184,6 @@ int main(int argc, char ** argv) {return 1;}- if (!params.mmproj.empty() &&- (!params.image.empty() ||- params.prompt.find("<img src=\"") != std::string::npos)) {- return llava_cli(argc, argv, ¶ms);- }-- // TODO: Dump params ?- //LOG("Params perplexity: %s\n", LOG_TOSTR(params.perplexity));-- // save choice to use color for later- // (note for later: this is a slightly awkward choice)- console::init(params.simple_io, params.use_color);- atexit([]() { console::cleanup(); });-if (!FLAG_unsecure && !llamafile_has_gpu()) {// Enable pledge() security on Linux and OpenBSD.// - We do this *after* opening the log file for writing.@@ -213,6 +199,7 @@ int main(int argc, char ** argv) {} else {promises = "stdio rpath tty";}+ __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;if (pledge(0, 0)) {LOG_TEE("warning: this OS doesn't support pledge() security\n");} else if (pledge(promises, 0)) {@@ -221,6 +208,20 @@ int main(int argc, char ** argv) {}}+ if (!params.mmproj.empty() &&+ (!params.image.empty() ||+ params.prompt.find("<img src=\"") != std::string::npos)) {+ return llava_cli(argc, argv, ¶ms);+ }++ // TODO: Dump params ?+ //LOG("Params perplexity: %s\n", LOG_TOSTR(params.perplexity));++ // save choice to use color for later+ // (note for later: this is a slightly awkward choice)+ console::init(!params.interactive || params.simple_io, params.use_color);+ atexit([]() { console::cleanup(); });+if (params.logits_all) {printf("\n************\n");printf("%s: please use the 'perplexity' tool for perplexity calculations\n", __func__);diff --git a/llamafile/debug.cpp b/llamafile/debug.cppnew file mode 100644index 00000000..2a5a6fc6--- /dev/null+++ b/llamafile/debug.cpp@@ -0,0 +1,197 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llamafile.h"++#include <cosmo.h>+#include <fenv.h>+#include <signal.h>+#include <stdatomic.h>+#include <termios.h>+#include <unistd.h>++#include "llama.cpp/ggml.h"++#define CONFIG FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW // | FE_UNDERFLOW++bool FLAG_trap;+thread_local int llamafile_debug_op_index;+const struct ggml_cgraph *llamafile_debug_graph;++static struct TerminalBuddy {+ TerminalBuddy() {+ active = !tcgetattr(0, &state);+ }+ void restore() {+ if (active)+ tcsetattr(0, TCSANOW, &state);+ }+ bool active;+ struct termios state;+} g_terminal_buddy;++static const char *describe_vertex(struct ggml_tensor *t) {+ if (t->op == GGML_OP_UNARY)+ return ggml_unary_op_name((enum ggml_unary_op)t->op_params[0]);+ if (t->op)+ return ggml_op_name(t->op);+ return t->name;+}++static void print_graph(FILE *f, const struct ggml_cgraph *g) {+ for (int i = 0; i < g->n_nodes; ++i) {+ fprintf(f, "%5d %p %s:%s(", i, g->nodes[i], ggml_type_name(g->nodes[i]->type),+ describe_vertex(g->nodes[i]));+ for (int j = 0; j < GGML_MAX_SRC && g->nodes[i]->src[j]; ++j) {+ if (j)+ fprintf(f, ", ");+ fprintf(f, "%s:%s[%p]", ggml_type_name(g->nodes[i]->src[j]->type),+ describe_vertex(g->nodes[i]->src[j]), g->nodes[i]->src[j]);+ }+ fprintf(f, ")\n");+ }+}++// TODO: put this in cosmo libc+int feenableexcept(int excepts) {+ excepts &= FE_ALL_EXCEPT;+#ifdef __x86_64__+ unsigned short neux;+ asm("fstcw\t%0" : "=m"(neux));+ unsigned short res = ~neux & FE_ALL_EXCEPT;+ neux &= ~excepts;+ asm("fldcw\t%0" : /* no inputs */ : "m"(neux));+ unsigned neu;+ asm("stmxcsr\t%0" : "=m"(neu));+ neu &= ~(excepts << 7);+ asm("ldmxcsr\t%0" : /* no inputs */ : "m"(neu));+ return res;+#else+ unsigned fpcr;+ unsigned fpcr2;+ unsigned updated_fpcr;+ fpcr = __builtin_aarch64_get_fpcr();+ fpcr2 = fpcr | (excepts << 8);+ if (fpcr != fpcr2) {+ __builtin_aarch64_set_fpcr(fpcr2);+ // floating point exception trapping is optional in aarch64+ updated_fpcr = __builtin_aarch64_get_fpsr();+ if (fpcr2 & ~updated_fpcr)+ return -1;+ }+ return (fpcr >> 8) & FE_ALL_EXCEPT;+#endif+}++int fedisableexcept(int excepts) {+ excepts &= FE_ALL_EXCEPT;+#ifdef __x86_64__+ unsigned neu;+ unsigned short neux;+ asm("fstcw\t%0" : "=m"(neux));+ unsigned short res = ~neux & FE_ALL_EXCEPT;+ neux |= excepts;+ asm("fldcw\t%0" : /* no inputs */ : "m"(neux));+ asm("stmxcsr\t%0" : "=m"(neu));+ neu |= excepts << 7;+ asm("ldmxcsr\t%0" : /* no inputs */ : "m"(neu));+ return res;+#else+ unsigned fpcr;+ unsigned fpcr2;+ fpcr = __builtin_aarch64_get_fpcr();+ fpcr2 = fpcr & ~(excepts << 8);+ if (fpcr != fpcr2)+ __builtin_aarch64_set_fpcr(fpcr2);+ return (fpcr >> 8) & FE_ALL_EXCEPT;+#endif+}++static inline void spinlock(atomic_uint *lock) {+ int x;+ for (;;) {+ x = atomic_exchange_explicit(lock, 1, memory_order_acquire);+ if (!x)+ break;+ }+}++static void on_sigfpe(int sig, siginfo_t *si, void *ctx) {+ static atomic_uint lock;+ spinlock(&lock); // first thread to crash wins+ g_terminal_buddy.restore();+ const char *s = "\nerror: ";+ if (si->si_code == FPE_INTDIV)+ tinyprint(2, s, "integer divide by zero\n", NULL);+ else if (si->si_code == FPE_INTOVF)+ tinyprint(2, s, "integer overflow\n", NULL);+ else if (si->si_code == FPE_FLTDIV)+ tinyprint(2, s, "floating point divide by zero\n", NULL);+ else if (si->si_code == FPE_FLTOVF)+ tinyprint(2, s, "floating point overflow\n", NULL);+ else if (si->si_code == FPE_FLTUND)+ tinyprint(2, s, "floating point underflow\n", NULL);+ else if (si->si_code == FPE_FLTRES)+ tinyprint(2, s, "floating point inexact\n", NULL);+ else if (si->si_code == FPE_FLTINV)+ tinyprint(2, s, "invalid floating point operation\n", NULL);+ else if (si->si_code == FPE_FLTSUB)+ tinyprint(2, s, "subscript out of range\n", NULL);+ else+ tinyprint(2, s, "caught sigfpe\n", NULL);+ ShowBacktrace(2, 0);+#ifdef LLAMAFILE_DEBUG+ const struct ggml_cgraph *g;+ if ((g = llamafile_debug_graph)) {+ int idx = llamafile_debug_op_index;+ const char *path = "/tmp/cgraph.txt";+ fprintf(stderr, "while executing %s at index #%d of %s\n", //+ ggml_op_name(g->nodes[idx]->op), idx, path);+ FILE *f = fopen(path, "w");+ print_graph(f, g);+ fclose(f);+ }+#endif+ _exit(128 + SIGFPE);+}++static void setup_sigfpe(void) {+ struct sigaction sa;+ sigemptyset(&sa.sa_mask);+ sa.sa_flags = SA_SIGINFO;+ sa.sa_sigaction = on_sigfpe;+ if (sigaction(SIGFPE, &sa, 0)) {+ perror("sigaction(SIGFPE)");+ exit(1);+ }+}++int llamafile_trapping_enabled(int delta) {+ static atomic_uint once;+ static thread_local int g_enabled;+ bool was_enabled = g_enabled > 0;+ bool is_enabled = (g_enabled += delta) > 0;+ feclearexcept(FE_ALL_EXCEPT);+ if (is_enabled && !was_enabled) {+ cosmo_once(&once, setup_sigfpe);+ feenableexcept(CONFIG);+ }+ if (!is_enabled && was_enabled) {+ fedisableexcept(CONFIG);+ }+ return g_enabled;+}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 57426ed3..133f684f 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -6,6 +6,11 @@extern "C" {#endif+// TODO(jart): remove in favor of c11 threads.h+#if !defined(__cplusplus) && !defined(thread_local)+#define thread_local _Thread_local+#endif+struct llamafile;struct llamafile *llamafile_open_gguf(const char *, const char *);void llamafile_close(struct llamafile *);@@ -17,7 +22,6 @@ size_t llamafile_tell(struct llamafile *);size_t llamafile_size(struct llamafile *);FILE *llamafile_fp(struct llamafile *);-void llamafile_init(void);void llamafile_check_cpu(void);void llamafile_help(const char *);void llamafile_log_command(char *[]);@@ -29,6 +33,7 @@ void llamafile_schlep(const void *, size_t);void llamafile_get_app_dir(char *, size_t);void llamafile_launch_browser(const char *);+extern bool FLAG_trap;extern bool FLAG_unsecure;#define LLAMAFILE_GPU_ERROR -2@@ -58,6 +63,15 @@ bool llamafile_mixmul(struct ggml_compute_params *, const struct ggml_tensor *,size_t llamafile_mixmul_needs(const struct ggml_tensor *, const struct ggml_tensor *,const struct ggml_tensor *);+struct StackFrame;+struct ggml_cgraph;+int feenableexcept(int);+int fedisableexcept(int);+int llamafile_trapping_enabled(int);+void ShowBacktrace(int, const struct StackFrame *);+extern const struct ggml_cgraph *llamafile_debug_graph;+extern thread_local int llamafile_debug_op_index;+#ifdef __cplusplus}#endif