Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
--- a/llama.cpp/tools/server/server.cpp
+++ b/llama.cpp/tools/server/server.cpp
@@ -16,6 +16,7 @@
#include <atomic>
#include <clocale>
#include <exception>
+#include <functional>
#include <signal.h>
#include <thread> // for std::thread::hardware_concurrency
@@ -23,6 +24,13 @@
#include <windows.h>
#endif
+#ifdef COSMOCC
+#include <cerrno>
+#include <cstring>
+#include <cosmo.h>
+#include "llamafile.h"
+#endif
+
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
@@ -41,6 +49,12 @@ static inline void signal_handler(int signal) {
struct server_run_opts {
bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself
bool resolve_model = true; // download / resolve -hf and docker models; off when the caller did it already
+
+ // llamafile combined mode (TUI + server in one process), see server_main()
+ std::function<void(const std::string &)> on_ready; // the server is loaded and accepting requests
+ std::function<void(std::function<void()>)> on_shutdown_available; // receives a function that stops the server
+ char ** argv = nullptr; // the sandbox detects --rpc on argv, it has no params field
+ bool combined_mode() const { return on_ready || on_shutdown_available; }
};
// satisfies -Wmissing-declarations (used by llama command)
@@ -48,6 +62,31 @@ int llama_server(int argc, char ** argv);
// to be used via CLI (argc / argv are used by router mode only)
int llama_server(common_params & params, int argc, char ** argv);
+
+// Core server logic - can be called from llamafile main.cpp or standalone main()
+// Optional callbacks for combined mode:
+// on_ready: called when server is fully loaded and accepting requests
+// on_shutdown_available: provides a function to trigger server shutdown
+int server_main(int argc, char ** argv,
+ std::function<void(const std::string &)> on_ready,
+ std::function<void(std::function<void()>)> on_shutdown_available);
+
+#ifdef LLAMAFILE_TUI
+// By now the program can safely exit. In combined mode return normally instead,
+// so the caller can handle cleanup and join the TUI thread.
+static void server_tui_exit(const server_run_opts & opts) {
+ if (opts.combined_mode()) {
+ return;
+ }
+ common_log_flush(common_log_main());
+ // Metal backend has async logging that llama_synchronize() doesn't wait for.
+ // Without this delay, _exit() might truncate llama_memory_breakdown_print's output
+ sleep(1);
+ // Use _exit() to avoid Metal cleanup crash (dangling refs with TUI + Metal + server)
+ _exit(0);
+}
+#endif
+
void llama_server_terminate();
void llama_server_terminate() {
if (shutdown_handler) {
@@ -602,6 +641,9 @@ static int server_run_router(common_params & params, int argc, char ** argv, con
// when the HTTP server stops, clean up and exit
clean_up();
+#ifdef LLAMAFILE_TUI
+ server_tui_exit(opts);
+#endif
return 0;
}
@@ -655,6 +697,107 @@ static int server_run_instance(common_params & params, server_role & role, commo
llama_backend_free();
};
+#ifdef COSMOCC
+ // Sandbox (pledge()/unveil(), llamafile issue #930). The whole
+ // derivation lives in llamafile/sandbox.c so it stays unit-testable
+ // and off the vendored diff. Placement is constrained on every side:
+ // - after the -hf / docker model resolve above: downloads need connect()
+ // - after ctx_http.init(): the TLS certificate/key are read there
+ // - before ctx_http.start(): filters only cover threads created
+ // after installation, so the HTTP listener/pool must not exist
+ // - before load_model(): untrusted GGUF data is parsed sandboxed
+ //
+ // Combined mode hosts an in-process HTTP client (the TUI) that needs
+ // connect(), which the accept-only sandbox forbids, so it's skipped.
+ //
+ // --connect spawns llama-connect once the server listens, i.e. after the
+ // lock; the sandbox forbids exec, so it is skipped there too.
+ if (opts.combined_mode()) {
+ SRV_WRN("%s", "sandbox: disabled in combined mode; use --server for a sandboxed server\n");
+ } else if (params.server_connect) {
+ SRV_WRN("%s", "sandbox: disabled with --connect (it spawns llama-connect, which the sandbox forbids)\n");
+ } else {
+ // pledge() (no outbound network / writes / exec) is applied by
+ // default. unveil() path confinement is opt-in (--confine-reads),
+ // because a server opens files by path at request time (multimodal
+ // media, etc.) that can't be known when the ruleset locks.
+ //
+ // cosmo installs the SECCOMP/Landlock filters on the calling thread
+ // only (no TSYNC), so quiesce every pre-existing background thread
+ // across the call: the llama.cpp log worker (pause() joins it,
+ // resume() respawns it under the filter) and the stream-session GC
+ // started at the top of server_main() (stop joins it, start
+ // respawns it; no sessions exist yet, so stopping is a no-op
+ // beyond the join).
+ //
+ // When --confine-reads is on, unveil() confines reads to these
+ // directories, so collect every on-disk path the loader opens
+ // after the lock. Add to this list when llama.cpp gains a new file
+ // parameter (else it'd be denied under confinement). Paths read
+ // before the sandbox (ssl_file_key/cert in ctx_http.init(), -hf
+ // downloads in the model resolve above) need no entry.
+ std::vector<const char *> read_paths, rw_paths;
+ read_paths.push_back(params.model.path.c_str());
+ read_paths.push_back(params.mmproj.path.c_str());
+ read_paths.push_back(params.media_path.c_str()); // request-time media
+ read_paths.push_back(params.speculative.draft.mparams.path.c_str());
+ read_paths.push_back(params.speculative.ngram_cache.lookup_cache_static.c_str());
+ read_paths.push_back(params.public_path.c_str());
+ for (const auto & la : params.lora_adapters)
+ read_paths.push_back(la.path.c_str());
+ for (const auto & cv : params.control_vectors)
+ read_paths.push_back(cv.fname.c_str());
+ rw_paths.push_back(params.slot_save_path.c_str());
+ rw_paths.push_back(params.path_prompt_cache.c_str());
+ rw_paths.push_back(params.speculative.ngram_cache.lookup_cache_dynamic.c_str());
+
+ // Features that dial out at runtime need connect(), which the
+ // accept-only default forbids; relax to full sockets for them
+ // (still no writes/exec). --rpc has no params field -- its handler
+ // registers backends directly -- so detect it on argv/env.
+ const bool needs_outbound =
+ params.ui_mcp_proxy ||
+ !params.server_tools.empty() ||
+ (opts.argv && llamafile_has(opts.argv, "--rpc")) ||
+ (std::getenv("LLAMA_ARG_RPC") != nullptr);
+
+ llamafile_sandbox_spec spec = {
+ read_paths.data(), (int) read_paths.size(),
+ rw_paths.data(), (int) rw_paths.size(),
+ FLAG_confine_reads, needs_outbound,
+ };
+ // Bind the listening sockets now: pledge "anet" only covers
+ // IPv4/IPv6, so a Unix domain socket (--host /path.sock) can't be
+ // created once the sandbox is on, while accept() on one that is
+ // already bound stays allowed (llamafile issue #971). Binding
+ // spawns no threads; ctx_http.start() reuses the sockets.
+ if (!ctx_http.bind_socket()) {
+ clean_up();
+ SRV_ERR("%s", "exiting due to HTTP server error\n");
+ return 1;
+ }
+
+ char promises[64];
+ common_log_pause(common_log_main());
+ server_stream_session_manager_stop();
+ int sandbox = llamafile_sandbox_server(&spec, promises, sizeof(promises));
+ int sandbox_errno = errno; // capture before the respawns spawn threads
+ server_stream_session_manager_start();
+ common_log_resume(common_log_main());
+ if (sandbox == LLAMAFILE_SANDBOX_FAILED) {
+ SRV_ERR("sandbox: pledge failed: %s; retry with --unsecure to disable sandboxing\n",
+ strerror(sandbox_errno));
+ clean_up();
+ return 1;
+ } else if (llamafile_sandbox_is_active(sandbox)) {
+ SRV_INF("sandbox: pledge(\"%s\") %s\n", promises,
+ llamafile_sandbox_describe(sandbox));
+ } else {
+ SRV_WRN("sandbox: %s\n", llamafile_sandbox_describe(sandbox));
+ }
+ }
+#endif
+
// start the HTTP server before loading the model to be able to serve /health requests
if (!ctx_http.start()) {
clean_up();
@@ -687,6 +830,13 @@ static int server_run_instance(common_params & params, server_role & role, commo
ctx_server.terminate();
};
+ // Provide shutdown function to caller (for combined mode)
+ if (opts.on_shutdown_available) {
+ opts.on_shutdown_available([&ctx_server]() {
+ ctx_server.terminate();
+ });
+ }
+
if (opts.install_signal_handlers) {
server_install_signal_handlers();
}
@@ -699,6 +849,11 @@ static int server_run_instance(common_params & params, server_role & role, commo
return 1;
}
+ // Notify caller that server is ready (for combined mode TUI startup)
+ if (opts.on_ready) {
+ opts.on_ready(ctx_http.listening_addresses.front());
+ }
+
// optionally, notify router server that this instance is ready
std::thread monitor_thread;
if (role.is_managed_by_router()) {
@@ -725,6 +880,9 @@ static int server_run_instance(common_params & params, server_role & role, commo
common_memory_breakdown_print(ll_ctx);
}
+#ifdef LLAMAFILE_TUI
+ server_tui_exit(opts);
+#endif
return 0;
}
@@ -770,6 +928,14 @@ static int server_run(common_params & params, int argc, char ** argv, server_rol
}
int llama_server(int argc, char ** argv) {
+ // thin wrapper for the `llama` multiplexer command; combined-mode callbacks
+ // are only used by llamafile's own TUI, which calls server_main() directly.
+ return server_main(argc, argv, nullptr, nullptr);
+}
+
+int server_main(int argc, char ** argv,
+ std::function<void(const std::string &)> on_ready,
+ std::function<void(std::function<void()>)> on_shutdown_available) {
server_role role;
std::setlocale(LC_NUMERIC, "C");
@@ -782,6 +948,10 @@ int llama_server(int argc, char ** argv) {
// own arguments required by this example
common_params params;
+#ifdef COSMOCC
+ llamafile_has_metal(); // triggers Metal backend registration on macOS ARM64
+#endif
+
common_init();
// start the stream session manager GC right after common init, before any HTTP route can
@@ -797,7 +967,12 @@ int llama_server(int argc, char ** argv) {
llama_backend_init();
llama_numa_init(params.numa);
- const int result = server_run(params, argc, argv, role, server_run_opts{});
+ server_run_opts opts;
+ opts.on_ready = std::move(on_ready);
+ opts.on_shutdown_available = std::move(on_shutdown_available);
+ opts.argv = argv;
+
+ const int result = server_run(params, argc, argv, role, opts);
common_log_flush(common_log_main());
return result;
}
@@ -811,3 +986,40 @@ int llama_server(common_params & params, int argc, char ** argv) {
server_role role;
return server_run(params, argc, argv, role, opts);
}
+
+// Standalone entry point for llama-server executable
+// Not compiled when building as part of llamafile TUI (which has its own main)
+// Having this allows us to test cosmocc-compiled llama.cpp in isolation.
+#ifndef LLAMAFILE_TUI
+int main(int argc, char ** argv) {
+#ifdef COSMOCC
+ argc = cosmo_args("/zip/.args", &argv);
+
+ // Check if verbose mode is requested (must be set before GPU init)
+ bool verbose = llamafile_has(argv, "--verbose");
+ FLAG_verbose = verbose ? 1 : 0;
+
+ // --unsecure is llamafile's flag, not llama.cpp's: consume it here so
+ // common_params_parse() never sees it (llamafile's own main strips it
+ // via parse_llamafile_args instead)
+ if (llamafile_consume_flag(&argc, argv, "--unsecure"))
+ FLAG_unsecure = true;
+ if (llamafile_consume_flag(&argc, argv, "--confine-reads"))
+ FLAG_confine_reads = true;
+
+ // Initialize GPU support early (must happen BEFORE llama_backend_init())
+ // This triggers dynamic loading of GPU backends (CUDA, ROCm, Metal)
+ // The llamafile_has_* functions use lazy initialization via cosmo_once()
+ llamafile_has_gpu();
+ if (!verbose) {
+ // disable ggml verbose logging
+ if (llamafile_has_metal()) {
+ llamafile_metal_log_set(llamafile_log_callback_null, NULL);
+ } else if (llamafile_has_cuda() || llamafile_has_amd_gpu()) {
+ llamafile_cuda_log_set(llamafile_log_callback_null, NULL);
+ }
+ }
+#endif
+ return server_main(argc, argv, nullptr, nullptr);
+}
+#endif