Skip to content

Distribute and run LLMs with a single file.

Make stack overflows recoverable in new server

Justine Tunney7f099cd69e49
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index d420b0c2..b91cc467 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -42,6 +42,7 @@ SOFTWARE.");
#include "llamafile/debug.h"
#include "llamafile/sgemm.h"
#include "llamafile/thread.h"
+#include "llamafile/crash.h"
#include <alloca.h>
#include <assert.h>
@@ -18886,6 +18887,11 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
return 0;
}
+static thread_ret_t ggml_graph_compute_thread_callback(void * data) {
+ protect_against_stack_overflow(); // [jart]
+ return ggml_graph_compute_thread(data);
+}
+
struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threads) {
if (n_threads <= 0) {
n_threads = GGML_DEFAULT_N_THREADS;
@@ -19198,7 +19204,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
};
const int rc = ggml_thread_create((pthread_t *)&workers[j].thrd, NULL,
- ggml_graph_compute_thread, &workers[j]);
+ ggml_graph_compute_thread_callback, &workers[j]);
GGML_ASSERT(rc == 0);
UNUSED(rc);
}
diff --git a/llama.cpp/llama-bench/main.cpp b/llama.cpp/llama-bench/main.cpp
index 8064b294..a04961fd 100644
--- a/llama.cpp/llama-bench/main.cpp
+++ b/llama.cpp/llama-bench/main.cpp
@@ -111,8 +111,6 @@ static void launch_sigint_thread(void) {
sigfillset(&block_every_signal);
pthread_attr_t attr;
pthread_attr_init(&attr);
- pthread_attr_setstacksize(&attr, 65536);
- pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));
pthread_attr_setsigmask_np(&attr, &block_every_signal);
pthread_create(&th, &attr, safe_sigint_handler, 0);
pthread_attr_destroy(&attr);
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 44673872..e58714f3 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -58,7 +58,7 @@
#include <sys/resource.h>
#include <ctl/vector.h>
-#include "llamafile/server/threadlocal.h"
+#include "llamafile/threadlocal.h"
#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((__format__(__gnu_printf__, __VA_ARGS__)))
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 27ddb551..8dc7f0d3 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -111,8 +111,6 @@ static void launch_sigint_thread(void) {
sigfillset(&block_every_signal);
pthread_attr_t attr;
pthread_attr_init(&attr);
- pthread_attr_setstacksize(&attr, 65536);
- pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));
pthread_attr_setsigmask_np(&attr, &block_every_signal);
pthread_create(&th, &attr, safe_sigint_handler, 0);
pthread_attr_destroy(&attr);
diff --git a/llamafile/crash.h b/llamafile/crash.h
index 99922760..37d8aa19 100644
--- a/llamafile/crash.h
+++ b/llamafile/crash.h
@@ -9,6 +9,7 @@ struct StackFrame;
char *describe_crash(char *, size_t, int, siginfo_t *, void *);
char *describe_backtrace(char *, size_t, const struct StackFrame *);
+void protect_against_stack_overflow(void);
#ifdef __cplusplus
}
diff --git a/llamafile/protect_against_stack_overflow.cpp b/llamafile/protect_against_stack_overflow.cpp
new file mode 100644
index 00000000..e414a801
--- /dev/null
+++ b/llamafile/protect_against_stack_overflow.cpp
@@ -0,0 +1,47 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "crash.h"
+
+#include <pthread.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "threadlocal.h"
+
+// Call this from your thread start function
+// Your thread needs to have a guard page (default)
+// You also need SIGSEGV and SIGBUS handlers w/ SA_ONSTACK
+void protect_against_stack_overflow(void) {
+ static ThreadLocal<void> cleanup([](void *ptr) {
+ struct sigaltstack ss;
+ ss.ss_flags = SS_DISABLE;
+ ss.ss_size = 0;
+ ss.ss_sp = 0;
+ sigaltstack(&ss, 0);
+ free(ptr);
+ });
+ struct sigaltstack ss;
+ ss.ss_flags = 0;
+ ss.ss_size = sysconf(_SC_MINSIGSTKSZ) + 16384;
+ if ((ss.ss_sp = malloc(ss.ss_size))) {
+ if (sigaltstack(&ss, 0))
+ __builtin_trap();
+ cleanup.set(ss.ss_sp);
+ }
+}
diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cpp
index cd67ea3d..546dba45 100644
--- a/llamafile/server/client.cpp
+++ b/llamafile/server/client.cpp
@@ -27,10 +27,10 @@
#include "llama.cpp/llama.h"
#include "llamafile/llamafile.h"
+#include "llamafile/threadlocal.h"
#include "llamafile/version.h"
#include "log.h"
-#include "threadlocal.h"
#include "time.h"
#define STANDARD_RESPONSE_HEADERS \
diff --git a/llamafile/server/server.cpp b/llamafile/server/server.cpp
index ef596ce7..1bc5aaf4 100644
--- a/llamafile/server/server.cpp
+++ b/llamafile/server/server.cpp
@@ -25,6 +25,7 @@
#include <sys/socket.h>
#include <time.h>
+#include "llamafile/crash.h"
#include "llamafile/llamafile.h"
#include "log.h"
@@ -90,6 +91,7 @@ Server::close()
void*
worker_thread(void* arg)
{
+ protect_against_stack_overflow();
Worker* worker = (Worker*)arg;
worker->run();
return 0;
@@ -103,8 +105,6 @@ Server::spawn()
pthread_attr_t attr;
worker = new Worker(this);
pthread_attr_init(&attr);
- pthread_attr_setstacksize(&attr, 65536);
- pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
if ((err = pthread_create(&worker->th, &attr, worker_thread, worker)))
delete worker;
diff --git a/llamafile/server/signals.cpp b/llamafile/server/signals.cpp
index 64837884..1dd9da96 100644
--- a/llamafile/server/signals.cpp
+++ b/llamafile/server/signals.cpp
@@ -21,6 +21,7 @@
#include <ucontext.h>
#include "llamafile/crash.h"
+#include "llamafile/threadlocal.h"
#include "log.h"
#include "server.h"
@@ -82,11 +83,13 @@ signals_init(void)
sigaction(SIGABRT, &sa, &old.sigabrt);
sigaction(SIGTRAP, &sa, &old.sigtrap);
sigaction(SIGFPE, &sa, &old.sigfpe);
- sigaction(SIGBUS, &sa, &old.sigbus);
- sigaction(SIGSEGV, &sa, &old.sigsegv);
sigaction(SIGILL, &sa, &old.sigill);
sigaction(SIGXCPU, &sa, &old.sigxcpu);
sigaction(SIGXFSZ, &sa, &old.sigxfsz);
+
+ sa.sa_flags |= SA_ONSTACK;
+ sigaction(SIGBUS, &sa, &old.sigbus);
+ sigaction(SIGSEGV, &sa, &old.sigsegv);
}
void
diff --git a/llamafile/server/time.cpp b/llamafile/server/time.cpp
index de3a4895..3420bc0f 100644
--- a/llamafile/server/time.cpp
+++ b/llamafile/server/time.cpp
@@ -23,6 +23,8 @@
#include <sys/auxv.h>
#include <unistd.h>
+#include "llamafile/crash.h"
+
#include "log.h"
//
@@ -133,6 +135,7 @@ time_worker(void* arg)
sigaddset(&ss, SIGUSR1);
sigaddset(&ss, SIGALRM);
pthread_sigmask(SIG_SETMASK, &ss, 0);
+ protect_against_stack_overflow();
set_thread_name("localtime");
for (;;) {
sleep(10);
diff --git a/llamafile/server/tokenize.cpp b/llamafile/server/tokenize.cpp
index f953048f..71f5dd37 100644
--- a/llamafile/server/tokenize.cpp
+++ b/llamafile/server/tokenize.cpp
@@ -26,6 +26,7 @@
#include "cleanup.h"
#include "fastjson.h"
#include "log.h"
+#include "signals.h"
#include "utils.h"
bool
diff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cpp
index 18090b87..ac45aa52 100644
--- a/llamafile/server/worker.cpp
+++ b/llamafile/server/worker.cpp
@@ -22,11 +22,11 @@
#include <pthread.h>
#include "llamafile/llamafile.h"
+#include "llamafile/threadlocal.h"
#include "client.h"
#include "log.h"
#include "signals.h"
-#include "threadlocal.h"
Worker::Worker(Server* server) : server(server)
{
diff --git a/llamafile/server/threadlocal.h b/llamafile/threadlocal.h
similarity index 100%
rename from llamafile/server/threadlocal.h
rename to llamafile/threadlocal.h