Skip to content

Distribute and run LLMs with a single file.

Synchronize after every op

Justine Tunney9cf736355383
diff --git a/build/config.mk b/build/config.mk
index c2973a1b..35acbf72 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -13,8 +13,8 @@ MKDEPS = $(COSMOCC)/bin/mkdeps
INSTALL = install
ARFLAGS = rcsD
-CCFLAGS = -g -O3 -fexceptions
-CPPFLAGS_ = -iquote. -mcosmo -DGGML_MULTIPLATFORM -Wno-attributes
+CCFLAGS = -g -O3 -fexceptions -fsignaling-nans
+CPPFLAGS_ = -iquote. -mcosmo -DGGML_MULTIPLATFORM -Wno-attributes -DLLAMAFILE_DEBUG
TARGET_ARCH = -Xx86_64-mavx -Xx86_64-mtune=znver4
TMPDIR = o//tmp
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index f46ef6ae..0a777802 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -233,6 +233,12 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
if (arg == "--cli") {
return true;
}
+ if (arg == "--trap") {
+ FLAG_trap = true;
+ FLAG_unsecure = true; // for better backtraces
+ llamafile_trapping_enabled(+1);
+ return true;
+ }
if (arg == "--unsecure") {
FLAG_unsecure = true;
return true;
diff --git a/llama.cpp/console.cpp b/llama.cpp/console.cpp
index 999f8045..144f69f1 100644
--- a/llama.cpp/console.cpp
+++ b/llama.cpp/console.cpp
@@ -1,13 +1,15 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+
#include "console.h"
+
#include <vector>
#include <iostream>
-
#include <climits>
#include <sys/ioctl.h>
#include <unistd.h>
#include <wchar.h>
+#include <cosmo.h>
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
@@ -30,6 +32,7 @@ namespace console {
static bool advanced_display = false;
static bool simple_io = true;
+static bool should_close_tty = false;
static display_t current_display = reset;
static FILE* out = stdout;
static FILE* tty = nullptr;
@@ -40,19 +43,32 @@ static termios initial_state;
//
void init(bool use_simple_io, bool use_advanced_display) {
- advanced_display = use_advanced_display;
+ should_close_tty = false;
simple_io = use_simple_io;
+ advanced_display = use_advanced_display;
if (!simple_io) {
- struct termios new_termios;
- tcgetattr(STDIN_FILENO, &initial_state);
- new_termios = initial_state;
- new_termios.c_lflag &= ~(ICANON | ECHO);
- new_termios.c_cc[VMIN] = 1;
- new_termios.c_cc[VTIME] = 0;
- tcsetattr(STDIN_FILENO, TCSANOW, &new_termios);
tty = fopen("/dev/tty", "w+e");
+ if (tty) {
+ should_close_tty = true;
+ } else if (IsLinux() || IsOpenbsd()) {
+ // this could happen because pledge() blocked us
+ tty = fdopen(0, "w+e");
+ }
if (tty != nullptr) {
- out = tty;
+ if (!tcgetattr(fileno(tty), &initial_state)) {
+ out = tty;
+ struct termios new_termios = initial_state;
+ new_termios.c_lflag &= ~(ICANON | ECHO);
+ new_termios.c_cc[VMIN] = 1;
+ new_termios.c_cc[VTIME] = 0;
+ tcsetattr(fileno(tty), TCSANOW, &new_termios);
+ } else {
+ simple_io = true;
+ fclose(tty);
+ tty = 0;
+ }
+ } else {
+ simple_io = true;
}
}
setlocale(LC_ALL, "");
@@ -64,11 +80,14 @@ void cleanup() {
// Restore settings
if (!simple_io) {
if (tty != nullptr) {
- out = stdout;
- fclose(tty);
+ fflush(tty);
+ tcsetattr(fileno(tty), TCSANOW, &initial_state);
+ if (should_close_tty) {
+ fclose(tty);
+ }
tty = nullptr;
+ out = stdout;
}
- tcsetattr(STDIN_FILENO, TCSANOW, &initial_state);
}
}
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 194ecca6..510cca01 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -117,9 +117,9 @@ void ggml_print_backtrace(void) {
/*#define GGML_PERF*/
#define GGML_DEBUG 0
-#define GGML_GELU_FP16
-#define GGML_GELU_QUICK_FP16
-#define GGML_SILU_FP16
+// #define GGML_GELU_FP16
+// #define GGML_GELU_QUICK_FP16
+// #define GGML_SILU_FP16
// #define GGML_CROSS_ENTROPY_EXP_FP16
// #define GGML_FLASH_ATTN_EXP_FP16
@@ -2679,6 +2679,7 @@ static inline int ggml_up(int n, int m) {
struct ggml_context * ggml_init(struct ggml_init_params params) {
// make this function thread safe
ggml_critical_section_start();
+ llamafile_trapping_enabled(-1);
static bool is_first_call = true;
@@ -2750,6 +2751,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {
if (ctx == NULL) {
GGML_PRINT_DEBUG("%s: no unused context found\n", __func__);
+ llamafile_trapping_enabled(+1);
ggml_critical_section_end();
return NULL;
@@ -2781,6 +2783,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {
GGML_PRINT_DEBUG("%s: context initialized\n", __func__);
+ llamafile_trapping_enabled(+1);
ggml_critical_section_end();
return ctx;
@@ -7105,19 +7108,12 @@ void ggml_syncthreads(struct ggml_compute_params * params) {
// ops must call this before accessing wdata
// otherwise overlapping threads on previous op might clobber
void *ggml_acquire_wdata(struct ggml_compute_params * params) {
- if (params->limbo) {
- ggml_syncthreads(params);
- }
- params->limbo = true;
return params->wdata;
}
// ops should call this after writing to wdata before reading
void ggml_release_wdata(struct ggml_compute_params * params) {
- if (params->limbo) {
- ggml_syncthreads(params);
- params->limbo = false;
- }
+ ggml_syncthreads(params);
}
////////////////////////////////////////////////////////////////////////////////
@@ -11531,12 +11527,9 @@ static void ggml_compute_forward_set_f32(
if (!inplace) {
if (!params->ith) {
- // memcpy needs to be synchronized across threads to avoid race conditions.
- // => do it in INIT phase
- memcpy(
- ((char *) dst->data),
- ((char *) src0->data),
- ggml_nbytes(dst));
+ memcpy(((char *) dst->data),
+ ((char *) src0->data),
+ ggml_nbytes(dst));
}
ggml_syncthreads(params);
}
@@ -17759,35 +17752,6 @@ struct ggml_compute_state {
enum ggml_status ec;
};
-// returns true if `src` is direct dependency of `node`
-static bool ggml_has_src(const struct ggml_tensor * node,
- const struct ggml_tensor * src) {
- for (int i = 0; i < GGML_MAX_SRC; ++i) {
- if (node->src[i] == src) {
- return true;
- }
- }
- return false;
-}
-
-// returns true if `dest` depends on any outputs since `mark`
-//
-// - cgraph->nodes[dest] is about to be executed
-// - syncthreads() last happened right before cgraph->nodes[mark] executed
-// - syncthreads() has 1.8 µs overhead minimum on Apple M2 when nth == 12
-//
-static bool ggml_needs_barrier(const struct ggml_cgraph * cgraph, int mark, int dest) {
- assert(mark >= 0);
- assert(mark <= dest);
- assert(dest < cgraph->n_nodes);
- for (; mark < dest; ++mark) {
- if (ggml_has_src(cgraph->nodes[dest], cgraph->nodes[mark])) {
- return true;
- }
- }
- return false;
-}
-
static thread_ret_t ggml_graph_compute_thread(void * data) {
struct ggml_compute_state * state = (struct ggml_compute_state *) data;
const struct ggml_cgraph * cgraph = state->shared->cgraph;
@@ -17799,11 +17763,16 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
/*.wsize =*/ cplan->work_size,
/*.wdata =*/ cplan->work_data,
/*.barrier =*/ &state->shared->barrier,
- /*.limbo =*/ false,
};
set_numa_thread_affinity(state->ith);
+#ifdef LLAMAFILE_DEBUG
+ if (FLAG_trap) {
+ llamafile_trapping_enabled(+1);
+ }
+#endif
+
#ifdef GGML_PERF
int64_t start_cycles, start_time_us;
if (!state->ith) {
@@ -17819,17 +17788,19 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
return 0;
}
- if (ggml_needs_barrier(cgraph, mark, node_n)) {
- ggml_syncthreads(&params);
- mark = node_n;
- }
+#ifdef LLAMAFILE_DEBUG
+ llamafile_debug_op_index = node_n;
+#endif
struct ggml_tensor *node = cgraph->nodes[node_n];
params.nth = state->shared->n_threads;
ggml_compute_forward(&params, node);
+ // this barrier could potentially be eliminated in 15%+ of cases
+ // however, it would give rise to ghoulish errors w/ little gain
+ ggml_syncthreads(&params);
+
#if GGML_PERF
- ggml_syncthreads(&state->shared->barrier);
if (!state->ith) {
int64_t end_cycles = ggml_perf_cycles();
int64_t end_time_us = ggml_perf_time_us();
@@ -17844,8 +17815,6 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
#endif
}
- ggml_syncthreads(&params);
-
return 0;
}
@@ -18075,6 +18044,10 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
};
struct ggml_compute_state * workers = alloca(sizeof(struct ggml_compute_state)*n_threads);
+#ifdef LLAMAFILE_DEBUG
+ llamafile_debug_graph = cgraph;
+#endif
+
// create thread pool
if (n_threads > 1) {
for (int j = 1; j < n_threads; ++j) {
@@ -18134,6 +18107,10 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
// fprintf(stderr, "%6d barriers %6d ops\n", state_shared.barrier.phase, cgraph->n_nodes);
+#ifdef LLAMAFILE_DEBUG
+ llamafile_debug_graph = 0;
+#endif
+
return compute_status;
}
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index eb39dbc7..ff93b62f 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -678,7 +678,6 @@ extern "C" {
void * wdata;
struct ggml_barrier * barrier;
- bool limbo;
};
GGML_API void ggml_syncthreads (struct ggml_compute_params *);
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 2d641f76..84388b1e 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -184,20 +184,6 @@ int main(int argc, char ** argv) {
return 1;
}
- if (!params.mmproj.empty() &&
- (!params.image.empty() ||
- params.prompt.find("<img src=\"") != std::string::npos)) {
- return llava_cli(argc, argv, &params);
- }
-
- // TODO: Dump params ?
- //LOG("Params perplexity: %s\n", LOG_TOSTR(params.perplexity));
-
- // save choice to use color for later
- // (note for later: this is a slightly awkward choice)
- console::init(params.simple_io, params.use_color);
- atexit([]() { console::cleanup(); });
-
if (!FLAG_unsecure && !llamafile_has_gpu()) {
// Enable pledge() security on Linux and OpenBSD.
// - We do this *after* opening the log file for writing.
@@ -213,6 +199,7 @@ int main(int argc, char ** argv) {
} else {
promises = "stdio rpath tty";
}
+ __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;
if (pledge(0, 0)) {
LOG_TEE("warning: this OS doesn't support pledge() security\n");
} else if (pledge(promises, 0)) {
@@ -221,6 +208,20 @@ int main(int argc, char ** argv) {
}
}
+ if (!params.mmproj.empty() &&
+ (!params.image.empty() ||
+ params.prompt.find("<img src=\"") != std::string::npos)) {
+ return llava_cli(argc, argv, &params);
+ }
+
+ // TODO: Dump params ?
+ //LOG("Params perplexity: %s\n", LOG_TOSTR(params.perplexity));
+
+ // save choice to use color for later
+ // (note for later: this is a slightly awkward choice)
+ console::init(!params.interactive || params.simple_io, params.use_color);
+ atexit([]() { console::cleanup(); });
+
if (params.logits_all) {
printf("\n************\n");
printf("%s: please use the 'perplexity' tool for perplexity calculations\n", __func__);
diff --git a/llamafile/debug.cpp b/llamafile/debug.cpp
new file mode 100644
index 00000000..2a5a6fc6
--- /dev/null
+++ b/llamafile/debug.cpp
@@ -0,0 +1,197 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "llamafile.h"
+
+#include <cosmo.h>
+#include <fenv.h>
+#include <signal.h>
+#include <stdatomic.h>
+#include <termios.h>
+#include <unistd.h>
+
+#include "llama.cpp/ggml.h"
+
+#define CONFIG FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW // | FE_UNDERFLOW
+
+bool FLAG_trap;
+thread_local int llamafile_debug_op_index;
+const struct ggml_cgraph *llamafile_debug_graph;
+
+static struct TerminalBuddy {
+ TerminalBuddy() {
+ active = !tcgetattr(0, &state);
+ }
+ void restore() {
+ if (active)
+ tcsetattr(0, TCSANOW, &state);
+ }
+ bool active;
+ struct termios state;
+} g_terminal_buddy;
+
+static const char *describe_vertex(struct ggml_tensor *t) {
+ if (t->op == GGML_OP_UNARY)
+ return ggml_unary_op_name((enum ggml_unary_op)t->op_params[0]);
+ if (t->op)
+ return ggml_op_name(t->op);
+ return t->name;
+}
+
+static void print_graph(FILE *f, const struct ggml_cgraph *g) {
+ for (int i = 0; i < g->n_nodes; ++i) {
+ fprintf(f, "%5d %p %s:%s(", i, g->nodes[i], ggml_type_name(g->nodes[i]->type),
+ describe_vertex(g->nodes[i]));
+ for (int j = 0; j < GGML_MAX_SRC && g->nodes[i]->src[j]; ++j) {
+ if (j)
+ fprintf(f, ", ");
+ fprintf(f, "%s:%s[%p]", ggml_type_name(g->nodes[i]->src[j]->type),
+ describe_vertex(g->nodes[i]->src[j]), g->nodes[i]->src[j]);
+ }
+ fprintf(f, ")\n");
+ }
+}
+
+// TODO: put this in cosmo libc
+int feenableexcept(int excepts) {
+ excepts &= FE_ALL_EXCEPT;
+#ifdef __x86_64__
+ unsigned short neux;
+ asm("fstcw\t%0" : "=m"(neux));
+ unsigned short res = ~neux & FE_ALL_EXCEPT;
+ neux &= ~excepts;
+ asm("fldcw\t%0" : /* no inputs */ : "m"(neux));
+ unsigned neu;
+ asm("stmxcsr\t%0" : "=m"(neu));
+ neu &= ~(excepts << 7);
+ asm("ldmxcsr\t%0" : /* no inputs */ : "m"(neu));
+ return res;
+#else
+ unsigned fpcr;
+ unsigned fpcr2;
+ unsigned updated_fpcr;
+ fpcr = __builtin_aarch64_get_fpcr();
+ fpcr2 = fpcr | (excepts << 8);
+ if (fpcr != fpcr2) {
+ __builtin_aarch64_set_fpcr(fpcr2);
+ // floating point exception trapping is optional in aarch64
+ updated_fpcr = __builtin_aarch64_get_fpsr();
+ if (fpcr2 & ~updated_fpcr)
+ return -1;
+ }
+ return (fpcr >> 8) & FE_ALL_EXCEPT;
+#endif
+}
+
+int fedisableexcept(int excepts) {
+ excepts &= FE_ALL_EXCEPT;
+#ifdef __x86_64__
+ unsigned neu;
+ unsigned short neux;
+ asm("fstcw\t%0" : "=m"(neux));
+ unsigned short res = ~neux & FE_ALL_EXCEPT;
+ neux |= excepts;
+ asm("fldcw\t%0" : /* no inputs */ : "m"(neux));
+ asm("stmxcsr\t%0" : "=m"(neu));
+ neu |= excepts << 7;
+ asm("ldmxcsr\t%0" : /* no inputs */ : "m"(neu));
+ return res;
+#else
+ unsigned fpcr;
+ unsigned fpcr2;
+ fpcr = __builtin_aarch64_get_fpcr();
+ fpcr2 = fpcr & ~(excepts << 8);
+ if (fpcr != fpcr2)
+ __builtin_aarch64_set_fpcr(fpcr2);
+ return (fpcr >> 8) & FE_ALL_EXCEPT;
+#endif
+}
+
+static inline void spinlock(atomic_uint *lock) {
+ int x;
+ for (;;) {
+ x = atomic_exchange_explicit(lock, 1, memory_order_acquire);
+ if (!x)
+ break;
+ }
+}
+
+static void on_sigfpe(int sig, siginfo_t *si, void *ctx) {
+ static atomic_uint lock;
+ spinlock(&lock); // first thread to crash wins
+ g_terminal_buddy.restore();
+ const char *s = "\nerror: ";
+ if (si->si_code == FPE_INTDIV)
+ tinyprint(2, s, "integer divide by zero\n", NULL);
+ else if (si->si_code == FPE_INTOVF)
+ tinyprint(2, s, "integer overflow\n", NULL);
+ else if (si->si_code == FPE_FLTDIV)
+ tinyprint(2, s, "floating point divide by zero\n", NULL);
+ else if (si->si_code == FPE_FLTOVF)
+ tinyprint(2, s, "floating point overflow\n", NULL);
+ else if (si->si_code == FPE_FLTUND)
+ tinyprint(2, s, "floating point underflow\n", NULL);
+ else if (si->si_code == FPE_FLTRES)
+ tinyprint(2, s, "floating point inexact\n", NULL);
+ else if (si->si_code == FPE_FLTINV)
+ tinyprint(2, s, "invalid floating point operation\n", NULL);
+ else if (si->si_code == FPE_FLTSUB)
+ tinyprint(2, s, "subscript out of range\n", NULL);
+ else
+ tinyprint(2, s, "caught sigfpe\n", NULL);
+ ShowBacktrace(2, 0);
+#ifdef LLAMAFILE_DEBUG
+ const struct ggml_cgraph *g;
+ if ((g = llamafile_debug_graph)) {
+ int idx = llamafile_debug_op_index;
+ const char *path = "/tmp/cgraph.txt";
+ fprintf(stderr, "while executing %s at index #%d of %s\n", //
+ ggml_op_name(g->nodes[idx]->op), idx, path);
+ FILE *f = fopen(path, "w");
+ print_graph(f, g);
+ fclose(f);
+ }
+#endif
+ _exit(128 + SIGFPE);
+}
+
+static void setup_sigfpe(void) {
+ struct sigaction sa;
+ sigemptyset(&sa.sa_mask);
+ sa.sa_flags = SA_SIGINFO;
+ sa.sa_sigaction = on_sigfpe;
+ if (sigaction(SIGFPE, &sa, 0)) {
+ perror("sigaction(SIGFPE)");
+ exit(1);
+ }
+}
+
+int llamafile_trapping_enabled(int delta) {
+ static atomic_uint once;
+ static thread_local int g_enabled;
+ bool was_enabled = g_enabled > 0;
+ bool is_enabled = (g_enabled += delta) > 0;
+ feclearexcept(FE_ALL_EXCEPT);
+ if (is_enabled && !was_enabled) {
+ cosmo_once(&once, setup_sigfpe);
+ feenableexcept(CONFIG);
+ }
+ if (!is_enabled && was_enabled) {
+ fedisableexcept(CONFIG);
+ }
+ return g_enabled;
+}
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 57426ed3..133f684f 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -6,6 +6,11 @@
extern "C" {
#endif
+// TODO(jart): remove in favor of c11 threads.h
+#if !defined(__cplusplus) && !defined(thread_local)
+#define thread_local _Thread_local
+#endif
+
struct llamafile;
struct llamafile *llamafile_open_gguf(const char *, const char *);
void llamafile_close(struct llamafile *);
@@ -17,7 +22,6 @@ size_t llamafile_tell(struct llamafile *);
size_t llamafile_size(struct llamafile *);
FILE *llamafile_fp(struct llamafile *);
-void llamafile_init(void);
void llamafile_check_cpu(void);
void llamafile_help(const char *);
void llamafile_log_command(char *[]);
@@ -29,6 +33,7 @@ void llamafile_schlep(const void *, size_t);
void llamafile_get_app_dir(char *, size_t);
void llamafile_launch_browser(const char *);
+extern bool FLAG_trap;
extern bool FLAG_unsecure;
#define LLAMAFILE_GPU_ERROR -2
@@ -58,6 +63,15 @@ bool llamafile_mixmul(struct ggml_compute_params *, const struct ggml_tensor *,
size_t llamafile_mixmul_needs(const struct ggml_tensor *, const struct ggml_tensor *,
const struct ggml_tensor *);
+struct StackFrame;
+struct ggml_cgraph;
+int feenableexcept(int);
+int fedisableexcept(int);
+int llamafile_trapping_enabled(int);
+void ShowBacktrace(int, const struct StackFrame *);
+extern const struct ggml_cgraph *llamafile_debug_graph;
+extern thread_local int llamafile_debug_op_index;
+
#ifdef __cplusplus
}
#endif