Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make stack overflows recoverable in new server
Justine Tunney7f099cd69e49
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex d420b0c2..b91cc467 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -42,6 +42,7 @@ SOFTWARE.");#include "llamafile/debug.h"#include "llamafile/sgemm.h"#include "llamafile/thread.h"+#include "llamafile/crash.h"#include <alloca.h>#include <assert.h>@@ -18886,6 +18887,11 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {return 0;}+static thread_ret_t ggml_graph_compute_thread_callback(void * data) {+ protect_against_stack_overflow(); // [jart]+ return ggml_graph_compute_thread(data);+}+struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threads) {if (n_threads <= 0) {n_threads = GGML_DEFAULT_N_THREADS;@@ -19198,7 +19204,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl};const int rc = ggml_thread_create((pthread_t *)&workers[j].thrd, NULL,- ggml_graph_compute_thread, &workers[j]);+ ggml_graph_compute_thread_callback, &workers[j]);GGML_ASSERT(rc == 0);UNUSED(rc);}diff --git a/llama.cpp/llama-bench/main.cpp b/llama.cpp/llama-bench/main.cppindex 8064b294..a04961fd 100644--- a/llama.cpp/llama-bench/main.cpp+++ b/llama.cpp/llama-bench/main.cpp@@ -111,8 +111,6 @@ static void launch_sigint_thread(void) {sigfillset(&block_every_signal);pthread_attr_t attr;pthread_attr_init(&attr);- pthread_attr_setstacksize(&attr, 65536);- pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));pthread_attr_setsigmask_np(&attr, &block_every_signal);pthread_create(&th, &attr, safe_sigint_handler, 0);pthread_attr_destroy(&attr);diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 44673872..e58714f3 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -58,7 +58,7 @@#include <sys/resource.h>#include <ctl/vector.h>-#include "llamafile/server/threadlocal.h"+#include "llamafile/threadlocal.h"#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((__format__(__gnu_printf__, __VA_ARGS__)))diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 27ddb551..8dc7f0d3 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -111,8 +111,6 @@ static void launch_sigint_thread(void) {sigfillset(&block_every_signal);pthread_attr_t attr;pthread_attr_init(&attr);- pthread_attr_setstacksize(&attr, 65536);- pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));pthread_attr_setsigmask_np(&attr, &block_every_signal);pthread_create(&th, &attr, safe_sigint_handler, 0);pthread_attr_destroy(&attr);diff --git a/llamafile/crash.h b/llamafile/crash.hindex 99922760..37d8aa19 100644--- a/llamafile/crash.h+++ b/llamafile/crash.h@@ -9,6 +9,7 @@ struct StackFrame;char *describe_crash(char *, size_t, int, siginfo_t *, void *);char *describe_backtrace(char *, size_t, const struct StackFrame *);+void protect_against_stack_overflow(void);#ifdef __cplusplus}diff --git a/llamafile/protect_against_stack_overflow.cpp b/llamafile/protect_against_stack_overflow.cppnew file mode 100644index 00000000..e414a801--- /dev/null+++ b/llamafile/protect_against_stack_overflow.cpp@@ -0,0 +1,47 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "crash.h"++#include <pthread.h>+#include <signal.h>+#include <stdlib.h>+#include <unistd.h>++#include "threadlocal.h"++// Call this from your thread start function+// Your thread needs to have a guard page (default)+// You also need SIGSEGV and SIGBUS handlers w/ SA_ONSTACK+void protect_against_stack_overflow(void) {+ static ThreadLocal<void> cleanup([](void *ptr) {+ struct sigaltstack ss;+ ss.ss_flags = SS_DISABLE;+ ss.ss_size = 0;+ ss.ss_sp = 0;+ sigaltstack(&ss, 0);+ free(ptr);+ });+ struct sigaltstack ss;+ ss.ss_flags = 0;+ ss.ss_size = sysconf(_SC_MINSIGSTKSZ) + 16384;+ if ((ss.ss_sp = malloc(ss.ss_size))) {+ if (sigaltstack(&ss, 0))+ __builtin_trap();+ cleanup.set(ss.ss_sp);+ }+}diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cppindex cd67ea3d..546dba45 100644--- a/llamafile/server/client.cpp+++ b/llamafile/server/client.cpp@@ -27,10 +27,10 @@#include "llama.cpp/llama.h"#include "llamafile/llamafile.h"+#include "llamafile/threadlocal.h"#include "llamafile/version.h"#include "log.h"-#include "threadlocal.h"#include "time.h"#define STANDARD_RESPONSE_HEADERS \diff --git a/llamafile/server/server.cpp b/llamafile/server/server.cppindex ef596ce7..1bc5aaf4 100644--- a/llamafile/server/server.cpp+++ b/llamafile/server/server.cpp@@ -25,6 +25,7 @@#include <sys/socket.h>#include <time.h>+#include "llamafile/crash.h"#include "llamafile/llamafile.h"#include "log.h"@@ -90,6 +91,7 @@ Server::close()void*worker_thread(void* arg){+ protect_against_stack_overflow();Worker* worker = (Worker*)arg;worker->run();return 0;@@ -103,8 +105,6 @@ Server::spawn()pthread_attr_t attr;worker = new Worker(this);pthread_attr_init(&attr);- pthread_attr_setstacksize(&attr, 65536);- pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);if ((err = pthread_create(&worker->th, &attr, worker_thread, worker)))delete worker;diff --git a/llamafile/server/signals.cpp b/llamafile/server/signals.cppindex 64837884..1dd9da96 100644--- a/llamafile/server/signals.cpp+++ b/llamafile/server/signals.cpp@@ -21,6 +21,7 @@#include <ucontext.h>#include "llamafile/crash.h"+#include "llamafile/threadlocal.h"#include "log.h"#include "server.h"@@ -82,11 +83,13 @@ signals_init(void)sigaction(SIGABRT, &sa, &old.sigabrt);sigaction(SIGTRAP, &sa, &old.sigtrap);sigaction(SIGFPE, &sa, &old.sigfpe);- sigaction(SIGBUS, &sa, &old.sigbus);- sigaction(SIGSEGV, &sa, &old.sigsegv);sigaction(SIGILL, &sa, &old.sigill);sigaction(SIGXCPU, &sa, &old.sigxcpu);sigaction(SIGXFSZ, &sa, &old.sigxfsz);++ sa.sa_flags |= SA_ONSTACK;+ sigaction(SIGBUS, &sa, &old.sigbus);+ sigaction(SIGSEGV, &sa, &old.sigsegv);}voiddiff --git a/llamafile/server/time.cpp b/llamafile/server/time.cppindex de3a4895..3420bc0f 100644--- a/llamafile/server/time.cpp+++ b/llamafile/server/time.cpp@@ -23,6 +23,8 @@#include <sys/auxv.h>#include <unistd.h>+#include "llamafile/crash.h"+#include "log.h"//@@ -133,6 +135,7 @@ time_worker(void* arg)sigaddset(&ss, SIGUSR1);sigaddset(&ss, SIGALRM);pthread_sigmask(SIG_SETMASK, &ss, 0);+ protect_against_stack_overflow();set_thread_name("localtime");for (;;) {sleep(10);diff --git a/llamafile/server/tokenize.cpp b/llamafile/server/tokenize.cppindex f953048f..71f5dd37 100644--- a/llamafile/server/tokenize.cpp+++ b/llamafile/server/tokenize.cpp@@ -26,6 +26,7 @@#include "cleanup.h"#include "fastjson.h"#include "log.h"+#include "signals.h"#include "utils.h"booldiff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cppindex 18090b87..ac45aa52 100644--- a/llamafile/server/worker.cpp+++ b/llamafile/server/worker.cpp@@ -22,11 +22,11 @@#include <pthread.h>#include "llamafile/llamafile.h"+#include "llamafile/threadlocal.h"#include "client.h"#include "log.h"#include "signals.h"-#include "threadlocal.h"Worker::Worker(Server* server) : server(server){diff --git a/llamafile/server/threadlocal.h b/llamafile/threadlocal.hsimilarity index 100%rename from llamafile/server/threadlocal.hrename to llamafile/threadlocal.h