Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add sandboxing for the server on Apple Silicon Macs (#261)
Haik Aftandiliand8e58ed8bea8
diff --git a/llama.cpp/server/macsandbox.cpp b/llama.cpp/server/macsandbox.cppnew file mode 100644index 00000000..b3599723--- /dev/null+++ b/llama.cpp/server/macsandbox.cpp@@ -0,0 +1,101 @@+#include <string>+#include <stdio.h>+#include <assert.h>+#include <dlfcn.h>+#include "macsandbox.h"++// Platform sandboxing dylib+#define MACOS_SANDBOXING_DYLIB "/usr/lib/system/libsystem_sandbox.dylib"++// Paths used for self-tests+#define MACOS_SANDBOXING_TEST_WRITE_PATH "/private/tmp/foo"+#define MACOS_SANDBOXING_TEST_READ_PATH "/private/etc/ntp.conf" // a benign file+#define MACOS_SANDBOXING_TEST_EXEC_PATH "/bin/date"++// dlsym'd functions+typedef int (*sandbox_init_with_parameters_t)(const char*, uint64_t,+ const char* const[], char**);+typedef int (*sandbox_check_t)(pid_t pid, const char* operation, int type, ...);++//+// A deny-by-default policy. Denies connecting to services, exec, and+// file I/O. Applies to operations/connections not already established. This is+// a restrictive policy that depends on the application already opening files+// and establishing connections to services it needs. As new features are added,+// the policy may need to be changed to allow more functionality (if for example+// a new file must be opened after the sandbox is enabled.) Logging of sandbox+// exceptions is enabled via Console.app.+//+static const char policy[] = R"SANDBOX_LITERAL(+ (version 1)+ (deny default)+ (if (defined? 'process-info*)+ (deny process-info*))+ (if (defined? 'nvram*)+ (deny nvram*))+ (if (defined? 'iokit-get-properties)+ (deny iokit-get-properties))+)SANDBOX_LITERAL";++void get_dlerror(std::string& out)+{+ char* errorp = cosmo_dlerror();+ if (errorp) {+ out = errorp;+ }+}++//+// Load the sandboxing platform dylib and turn on sandboxing for this process+// using the above `policy`.+//+// Returns non-zero on failure and populates `error_str` with an error message+// when one is available.+//+int mac_sandbox_init(std::string& error_str)+{+ // dlopen the platform dylib for sandboxing+ void* sblib = cosmo_dlopen(MACOS_SANDBOXING_DYLIB, RTLD_LOCAL|RTLD_NOW);+ if (!sblib) {+ get_dlerror(error_str);+ return 1;+ }++ // Get the function that applies a sandbox policy to the current process+ int (*sandbox_init_with_parametersp)(const char*, uint64_t,+ const char* const[], char**);+ sandbox_init_with_parametersp = (sandbox_init_with_parameters_t)+ cosmo_dlsym(sblib, "sandbox_init_with_parameters");+ if (!sandbox_init_with_parametersp) {+ get_dlerror(error_str);+ return 1;+ }++ // Get the function that checks a policy is applied for a given PID+ int (*sandbox_checkp)(pid_t pid, const char* operation, int type, ...);+ sandbox_checkp = (sandbox_check_t)cosmo_dlsym(sblib, "sandbox_check");+ if (!sandbox_checkp) {+ get_dlerror(error_str);+ return 1;+ }++ // Apply the policy+ char* errorbufp = NULL;+ int rv = sandbox_init_with_parametersp(policy, 0, NULL, &errorbufp);+ if (rv != 0) {+ if (errorbufp) {+ error_str = errorbufp;+ }+ return rv;+ }++ // Test that a policy is applied+ assert(sandbox_checkp(getpid(), NULL, 0) == 1);++ // Basic tests to ensure the policy works+ assert(fopen(MACOS_SANDBOXING_TEST_WRITE_PATH, "w") == NULL);+ assert(fopen(MACOS_SANDBOXING_TEST_READ_PATH, "r") == NULL);+ assert(system(MACOS_SANDBOXING_TEST_EXEC_PATH) != 0);++ return 0;+}diff --git a/llama.cpp/server/macsandbox.h b/llama.cpp/server/macsandbox.hnew file mode 100644index 00000000..6591c5f1--- /dev/null+++ b/llama.cpp/server/macsandbox.h@@ -0,0 +1,14 @@+#ifndef INCLUDE_MACSANDBOX_H+#define INCLUDE_MACSANDBOX_H++#include <string>++//+// Turn on sandboxing for this process.+//+// Returns non-zero on failure and populates `error_str` with+// an error message when one is available.+//+int mac_sandbox_init(std::string& error_str);++#endif // INCLUDE_MACSANDBOX_Hdiff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex f72721f0..e10358b3 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -23,6 +23,7 @@#include "tool/args/args.h"#include "libc/dce.h"#include "oai.h"+#include "macsandbox.h"// increase max payload length to allow use of larger context size#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576@@ -2624,31 +2625,60 @@ int server_cli(int argc, char **argv)llamafile_launch_browser(url);}- if (!params.unsecure && !llamafile_has_gpu()) {- // Enables pledge() security on Linux and OpenBSD.- // - We do this *after* binding the server socket.- // - We do this *after* opening the log file for writing.- // - We do this *after* opening a tab in the user browser.- // - We do this *before* loading any weights or graphdefs.- // In effect, what this does is:- // - Filesystem access is disabled entirely (except ZipOS).- // - On Linux, network access is restricted to accept() only.- // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.- char promises[32];- if (IsOpenbsd()) {- strlcpy(promises, "stdio inet", sizeof(promises));- } else {- strlcpy(promises, "stdio anet", sizeof(promises));- }- if (!startswith(sparams.public_path.c_str(), "/zip/")) {- strlcat(promises, " rpath", sizeof(promises));- }- __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;- if (pledge(0, 0)) {- LOG_TEE("warning: this OS doesn't support pledge() security\n");- } else if (pledge(promises, 0)) {- perror("pledge");- exit(1);+ if (!sparams.unsecure) {+ if (IsXnu()) {+ // Cosmopolitan libc explicitly does not support cosmo_dlopen on x64+ // macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt+ // to enable sandboxing and ignore the failure on x64 macOS.+ // (Alternatively, the sandbox-exec(1) command could be used to+ // launch llamafile with a sandbox profile.) A goal of llamafile is+ // to work easily out-of-the-box and it already operates without a+ // sandbox on macOS so don't require --unsecure on x64 macOS to+ // workaround the failure. If support is added later, it will be+ // used without further changes. On Apple Silicon, if sandbox+ // initialization fails, exit and recommend using --unsecure to+ // disable sandboxing.+ std::string error_msg;+ int rv = mac_sandbox_init(error_msg);+ if (rv != 0) {+ fprintf(stderr, "Sandbox initialization failed:\n"+ "rv: %d, error: %s.\n"+ "Disable sandboxing with the --unsecure option.\n",+ rv, error_msg.c_str());+ if (IsXnuSilicon()) {+ exit(1);+ } else {+ LOG_WARNING("Sandbox initialization failed. Not supported "+ "on x64 macOS. Continuing.", {});+ }+ }+ } else if (!llamafile_has_gpu()) {+ printf("In the sandboxing block!\n");+ // Enables pledge() security on Linux and OpenBSD.+ // - We do this *after* binding the server socket.+ // - We do this *after* opening the log file for writing.+ // - We do this *after* opening a tab in the user browser.+ // - We do this *before* loading any weights or graphdefs.+ // In effect, what this does is:+ // - Filesystem access is disabled entirely (except ZipOS).+ // - On Linux, network access is restricted to accept() only.+ // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.+ char promises[32];+ if (IsOpenbsd()) {+ strlcpy(promises, "stdio inet", sizeof(promises));+ } else {+ strlcpy(promises, "stdio anet", sizeof(promises));+ }+ if (!startswith(sparams.public_path.c_str(), "/zip/")) {+ strlcat(promises, " rpath", sizeof(promises));+ }+ __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;+ if (pledge(0, 0)) {+ LOG_TEE("warning: this OS doesn't support pledge() security\n");+ } else if (pledge(promises, 0)) {+ perror("pledge");+ exit(1);+ }}}