Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
--- a/llama.cpp/tools/server/server-models.cpp
+++ b/llama.cpp/tools/server/server-models.cpp
@@ -34,6 +34,7 @@
#include <cstring>
#ifndef _WIN32
+#include <csignal>
extern char **environ;
#endif
@@ -51,7 +52,25 @@ static void request_child_exit(server_subproc & proc) {
// single-threaded, watching all child processes at once
struct server_monitor {
server_monitor(server_models & models) : models(models) {
- th = std::thread([this]() { run(); });
+ th = std::thread([this]() {
+ // Block every signal on this thread so its waits can't be
+ // interrupted with EINTR (libcxx rethrows EINTR from a condition
+ // variable as std::system_error and std::terminates the process).
+ // Keep synchronous fault signals deliverable so real crashes still
+ // surface. SIGINT/SIGTERM belong to the main thread.
+ sigset_t block_mask;
+ sigfillset(&block_mask);
+ sigdelset(&block_mask, SIGSEGV);
+ sigdelset(&block_mask, SIGBUS);
+ sigdelset(&block_mask, SIGILL);
+ sigdelset(&block_mask, SIGFPE);
+ sigdelset(&block_mask, SIGABRT);
+ sigdelset(&block_mask, SIGSYS);
+ sigdelset(&block_mask, SIGTRAP);
+ pthread_sigmask(SIG_BLOCK, &block_mask, nullptr);
+
+ run();
+ });
}
~server_monitor() {
@@ -947,13 +966,19 @@ void server_models::load_models() {
}
// wait for all targeted models to reach UNLOADED; cv.wait handles unlock/relock
- cv.wait(lk, [&]() {
+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug
+ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,
+ // causing condition_variable::wait() to throw ETIMEDOUT.
+ auto all_unloaded = [&]() {
for (const auto & name : to_unload) {
auto it = mapping.find(name);
if (it != mapping.end() && it->second.meta.is_running()) return false;
}
return true;
- });
+ };
+ while (!all_unloaded()) {
+ cv.wait_for(lk, std::chrono::seconds(30));
+ }
// erase models no longer in any source
for (auto it = mapping.begin(); it != mapping.end(); ) {
@@ -1158,7 +1183,12 @@ void server_models::load(const std::string & name, const load_options & opts) {
std::unique_lock<std::mutex> lk(mutex);
// edge case: block until any in-progress reload has finished so we always load
// against the freshest preset and a consistent mapping state
- cv.wait(lk, [this]() { return !is_reloading; });
+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug
+ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,
+ // causing condition_variable::wait() to throw ETIMEDOUT.
+ while (is_reloading) {
+ cv.wait_for(lk, std::chrono::seconds(30));
+ }
// a download over an existing entry is decided under the lock, so a load of the
// same model cannot start in between: a paused download or an idle cached model
@@ -1381,7 +1411,10 @@ void server_models::unload_all() {
}
}
// wait for every child to exit, the monitor force-kills the ones that ignore the exit command
- cv.wait(lk, [this, &lk]() {
+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug
+ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,
+ // causing condition_variable::wait() to throw ETIMEDOUT.
+ auto all_stopped = [this, &lk]() {
if (sched->is_measuring(lk)) {
return false;
}
@@ -1391,7 +1424,10 @@ void server_models::unload_all() {
}
}
return true;
- });
+ };
+ while (!all_stopped()) {
+ cv.wait_for(lk, std::chrono::seconds(30));
+ }
}
void server_models::update_status(const std::string & name, const update_status_args & args) {
@@ -1552,16 +1588,22 @@ void server_models::wait(const std::string & name, std::function<bool(const serv
}
void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string & name, std::function<bool(const server_model_meta &)> predicate) {
- cv.wait(lk, [this, &name, &predicate]() {
+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug
+ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,
+ // causing condition_variable::wait() to throw ETIMEDOUT.
+ while (true) {
auto it = mapping.find(name);
if (it != mapping.end()) {
- return predicate(it->second.meta);
-
+ if (predicate(it->second.meta)) {
+ break;
+ }
+ } else {
+ // model was removed from mapping by another code path (e.g. load_models()).
+ // nothing left to wait for - tell the caller to proceed.
+ break;
}
- // model was removed from mapping by another code path (e.g. load_models()).
- // nothing left to wait for - tell the caller to proceed.
- return true;
- });
+ cv.wait_for(lk, std::chrono::seconds(30));
+ }
}
bool server_models::ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop) {
@@ -2631,10 +2673,10 @@ server_http_proxy::server_http_proxy(
auto pipe = std::make_shared<server_pipe<msg_t>>();
if (scheme == "https") {
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+#ifdef CPPHTTPLIB_SSL_ENABLED
cli.reset(new httplib::SSLClient(host, port));
#else
- throw std::runtime_error("HTTPS requested but CPPHTTPLIB_OPENSSL_SUPPORT is not defined");
+ throw std::runtime_error("HTTPS requested but the server is built without SSL support");
#endif
}