Skip to content

Distribute and run LLMs with a single file.

13.0 KB
Raw Blame
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
--- a/llama.cpp/tools/server/server.cpp
+++ b/llama.cpp/tools/server/server.cpp
@@ -16,6 +16,7 @@
 #include <atomic>
 #include <clocale>
 #include <exception>
+#include <functional>
 #include <signal.h>
 #include <thread> // for std::thread::hardware_concurrency
 
@@ -23,6 +24,13 @@
 #include <windows.h>
 #endif
 
+#ifdef COSMOCC
+#include <cerrno>
+#include <cstring>
+#include <cosmo.h>
+#include "llamafile.h"
+#endif
+
 static std::function<void(int)> shutdown_handler;
 static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
 
@@ -41,6 +49,12 @@ static inline void signal_handler(int signal) {
 struct server_run_opts {
     bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself
     bool resolve_model           = true; // download / resolve -hf and docker models; off when the caller did it already
+
+    // llamafile combined mode (TUI + server in one process), see server_main()
+    std::function<void(const std::string &)>   on_ready;              // the server is loaded and accepting requests
+    std::function<void(std::function<void()>)> on_shutdown_available; // receives a function that stops the server
+    char ** argv = nullptr; // the sandbox detects --rpc on argv, it has no params field
+    bool combined_mode() const { return on_ready || on_shutdown_available; }
 };
 
 // satisfies -Wmissing-declarations (used by llama command)
@@ -48,6 +62,31 @@ int llama_server(int argc, char ** argv);
 
 // to be used via CLI (argc / argv are used by router mode only)
 int llama_server(common_params & params, int argc, char ** argv);
+
+// Core server logic - can be called from llamafile main.cpp or standalone main()
+// Optional callbacks for combined mode:
+//   on_ready: called when server is fully loaded and accepting requests
+//   on_shutdown_available: provides a function to trigger server shutdown
+int server_main(int argc, char ** argv,
+                std::function<void(const std::string &)> on_ready,
+                std::function<void(std::function<void()>)> on_shutdown_available);
+
+#ifdef LLAMAFILE_TUI
+// By now the program can safely exit. In combined mode return normally instead,
+// so the caller can handle cleanup and join the TUI thread.
+static void server_tui_exit(const server_run_opts & opts) {
+    if (opts.combined_mode()) {
+        return;
+    }
+    common_log_flush(common_log_main());
+    // Metal backend has async logging that llama_synchronize() doesn't wait for.
+    // Without this delay, _exit() might truncate llama_memory_breakdown_print's output
+    sleep(1);
+    // Use _exit() to avoid Metal cleanup crash (dangling refs with TUI + Metal + server)
+    _exit(0);
+}
+#endif
+
 void llama_server_terminate();
 void llama_server_terminate() {
     if (shutdown_handler) {
@@ -602,6 +641,9 @@ static int server_run_router(common_params & params, int argc, char ** argv, con
     // when the HTTP server stops, clean up and exit
     clean_up();
 
+#ifdef LLAMAFILE_TUI
+    server_tui_exit(opts);
+#endif
     return 0;
 }
 
@@ -655,6 +697,107 @@ static int server_run_instance(common_params & params, server_role & role, commo
         llama_backend_free();
     };
 
+#ifdef COSMOCC
+    // Sandbox (pledge()/unveil(), llamafile issue #930). The whole
+    // derivation lives in llamafile/sandbox.c so it stays unit-testable
+    // and off the vendored diff. Placement is constrained on every side:
+    //   - after the -hf / docker model resolve above: downloads need connect()
+    //   - after ctx_http.init(): the TLS certificate/key are read there
+    //   - before ctx_http.start(): filters only cover threads created
+    //     after installation, so the HTTP listener/pool must not exist
+    //   - before load_model(): untrusted GGUF data is parsed sandboxed
+    //
+    // Combined mode hosts an in-process HTTP client (the TUI) that needs
+    // connect(), which the accept-only sandbox forbids, so it's skipped.
+    //
+    // --connect spawns llama-connect once the server listens, i.e. after the
+    // lock; the sandbox forbids exec, so it is skipped there too.
+    if (opts.combined_mode()) {
+        SRV_WRN("%s", "sandbox: disabled in combined mode; use --server for a sandboxed server\n");
+    } else if (params.server_connect) {
+        SRV_WRN("%s", "sandbox: disabled with --connect (it spawns llama-connect, which the sandbox forbids)\n");
+    } else {
+        // pledge() (no outbound network / writes / exec) is applied by
+        // default. unveil() path confinement is opt-in (--confine-reads),
+        // because a server opens files by path at request time (multimodal
+        // media, etc.) that can't be known when the ruleset locks.
+        //
+        // cosmo installs the SECCOMP/Landlock filters on the calling thread
+        // only (no TSYNC), so quiesce every pre-existing background thread
+        // across the call: the llama.cpp log worker (pause() joins it,
+        // resume() respawns it under the filter) and the stream-session GC
+        // started at the top of server_main() (stop joins it, start
+        // respawns it; no sessions exist yet, so stopping is a no-op
+        // beyond the join).
+        //
+        // When --confine-reads is on, unveil() confines reads to these
+        // directories, so collect every on-disk path the loader opens
+        // after the lock. Add to this list when llama.cpp gains a new file
+        // parameter (else it'd be denied under confinement). Paths read
+        // before the sandbox (ssl_file_key/cert in ctx_http.init(), -hf
+        // downloads in the model resolve above) need no entry.
+        std::vector<const char *> read_paths, rw_paths;
+        read_paths.push_back(params.model.path.c_str());
+        read_paths.push_back(params.mmproj.path.c_str());
+        read_paths.push_back(params.media_path.c_str());  // request-time media
+        read_paths.push_back(params.speculative.draft.mparams.path.c_str());
+        read_paths.push_back(params.speculative.ngram_cache.lookup_cache_static.c_str());
+        read_paths.push_back(params.public_path.c_str());
+        for (const auto & la : params.lora_adapters)
+            read_paths.push_back(la.path.c_str());
+        for (const auto & cv : params.control_vectors)
+            read_paths.push_back(cv.fname.c_str());
+        rw_paths.push_back(params.slot_save_path.c_str());
+        rw_paths.push_back(params.path_prompt_cache.c_str());
+        rw_paths.push_back(params.speculative.ngram_cache.lookup_cache_dynamic.c_str());
+
+        // Features that dial out at runtime need connect(), which the
+        // accept-only default forbids; relax to full sockets for them
+        // (still no writes/exec). --rpc has no params field -- its handler
+        // registers backends directly -- so detect it on argv/env.
+        const bool needs_outbound =
+            params.ui_mcp_proxy ||
+            !params.server_tools.empty() ||
+            (opts.argv && llamafile_has(opts.argv, "--rpc")) ||
+            (std::getenv("LLAMA_ARG_RPC") != nullptr);
+
+        llamafile_sandbox_spec spec = {
+            read_paths.data(), (int) read_paths.size(),
+            rw_paths.data(), (int) rw_paths.size(),
+            FLAG_confine_reads, needs_outbound,
+        };
+        // Bind the listening sockets now: pledge "anet" only covers
+        // IPv4/IPv6, so a Unix domain socket (--host /path.sock) can't be
+        // created once the sandbox is on, while accept() on one that is
+        // already bound stays allowed (llamafile issue #971). Binding
+        // spawns no threads; ctx_http.start() reuses the sockets.
+        if (!ctx_http.bind_socket()) {
+            clean_up();
+            SRV_ERR("%s", "exiting due to HTTP server error\n");
+            return 1;
+        }
+
+        char promises[64];
+        common_log_pause(common_log_main());
+        server_stream_session_manager_stop();
+        int sandbox = llamafile_sandbox_server(&spec, promises, sizeof(promises));
+        int sandbox_errno = errno;  // capture before the respawns spawn threads
+        server_stream_session_manager_start();
+        common_log_resume(common_log_main());
+        if (sandbox == LLAMAFILE_SANDBOX_FAILED) {
+            SRV_ERR("sandbox: pledge failed: %s; retry with --unsecure to disable sandboxing\n",
+                    strerror(sandbox_errno));
+            clean_up();
+            return 1;
+        } else if (llamafile_sandbox_is_active(sandbox)) {
+            SRV_INF("sandbox: pledge(\"%s\") %s\n", promises,
+                    llamafile_sandbox_describe(sandbox));
+        } else {
+            SRV_WRN("sandbox: %s\n", llamafile_sandbox_describe(sandbox));
+        }
+    }
+#endif
+
     // start the HTTP server before loading the model to be able to serve /health requests
     if (!ctx_http.start()) {
         clean_up();
@@ -687,6 +830,13 @@ static int server_run_instance(common_params & params, server_role & role, commo
         ctx_server.terminate();
     };
 
+    // Provide shutdown function to caller (for combined mode)
+    if (opts.on_shutdown_available) {
+        opts.on_shutdown_available([&ctx_server]() {
+            ctx_server.terminate();
+        });
+    }
+
     if (opts.install_signal_handlers) {
         server_install_signal_handlers();
     }
@@ -699,6 +849,11 @@ static int server_run_instance(common_params & params, server_role & role, commo
         return 1;
     }
 
+    // Notify caller that server is ready (for combined mode TUI startup)
+    if (opts.on_ready) {
+        opts.on_ready(ctx_http.listening_addresses.front());
+    }
+
     // optionally, notify router server that this instance is ready
     std::thread monitor_thread;
     if (role.is_managed_by_router()) {
@@ -725,6 +880,9 @@ static int server_run_instance(common_params & params, server_role & role, commo
         common_memory_breakdown_print(ll_ctx);
     }
 
+#ifdef LLAMAFILE_TUI
+    server_tui_exit(opts);
+#endif
     return 0;
 }
 
@@ -770,6 +928,14 @@ static int server_run(common_params & params, int argc, char ** argv, server_rol
 }
 
 int llama_server(int argc, char ** argv) {
+    // thin wrapper for the `llama` multiplexer command; combined-mode callbacks
+    // are only used by llamafile's own TUI, which calls server_main() directly.
+    return server_main(argc, argv, nullptr, nullptr);
+}
+
+int server_main(int argc, char ** argv,
+                std::function<void(const std::string &)> on_ready,
+                std::function<void(std::function<void()>)> on_shutdown_available) {
     server_role role;
 
     std::setlocale(LC_NUMERIC, "C");
@@ -782,6 +948,10 @@ int llama_server(int argc, char ** argv) {
     // own arguments required by this example
     common_params params;
 
+#ifdef COSMOCC
+    llamafile_has_metal();  // triggers Metal backend registration on macOS ARM64
+#endif
+
     common_init();
 
     // start the stream session manager GC right after common init, before any HTTP route can
@@ -797,7 +967,12 @@ int llama_server(int argc, char ** argv) {
     llama_backend_init();
     llama_numa_init(params.numa);
 
-    const int result = server_run(params, argc, argv, role, server_run_opts{});
+    server_run_opts opts;
+    opts.on_ready              = std::move(on_ready);
+    opts.on_shutdown_available = std::move(on_shutdown_available);
+    opts.argv                  = argv;
+
+    const int result = server_run(params, argc, argv, role, opts);
     common_log_flush(common_log_main());
     return result;
 }
@@ -811,3 +986,40 @@ int llama_server(common_params & params, int argc, char ** argv) {
     server_role role;
     return server_run(params, argc, argv, role, opts);
 }
+
+// Standalone entry point for llama-server executable
+// Not compiled when building as part of llamafile TUI (which has its own main)
+// Having this allows us to test cosmocc-compiled llama.cpp in isolation.
+#ifndef LLAMAFILE_TUI
+int main(int argc, char ** argv) {
+#ifdef COSMOCC
+    argc = cosmo_args("/zip/.args", &argv);
+
+    // Check if verbose mode is requested (must be set before GPU init)
+    bool verbose = llamafile_has(argv, "--verbose");
+    FLAG_verbose = verbose ? 1 : 0;
+
+    // --unsecure is llamafile's flag, not llama.cpp's: consume it here so
+    // common_params_parse() never sees it (llamafile's own main strips it
+    // via parse_llamafile_args instead)
+    if (llamafile_consume_flag(&argc, argv, "--unsecure"))
+        FLAG_unsecure = true;
+    if (llamafile_consume_flag(&argc, argv, "--confine-reads"))
+        FLAG_confine_reads = true;
+
+    // Initialize GPU support early (must happen BEFORE llama_backend_init())
+    // This triggers dynamic loading of GPU backends (CUDA, ROCm, Metal)
+    // The llamafile_has_* functions use lazy initialization via cosmo_once()
+    llamafile_has_gpu();
+    if (!verbose) {
+        // disable ggml verbose logging
+        if (llamafile_has_metal()) {
+            llamafile_metal_log_set(llamafile_log_callback_null, NULL);
+        } else if (llamafile_has_cuda() || llamafile_has_amd_gpu()) {
+            llamafile_cuda_log_set(llamafile_log_callback_null, NULL);
+        }
+    }
+#endif
+    return server_main(argc, argv, nullptr, nullptr);
+}
+#endif