Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Modify server.cpp to open browser tab on startup
Justine Tunney042ba195be5a
diff --git a/build/config.mk b/build/config.mkindex 0f6448b7..c500d103 100644--- a/build/config.mk+++ b/build/config.mk@@ -23,6 +23,7 @@ endifLC_ALL = CSOURCE_DATE_EPOCH = 0+export MODEexport TMPDIRexport LC_ALLexport SOURCE_DATE_EPOCHdiff --git a/llama.cpp/ggml-file.c b/llama.cpp/ggml-file.cindex 6e0c86b4..7b33bd65 100644--- a/llama.cpp/ggml-file.c+++ b/llama.cpp/ggml-file.c@@ -1,9 +1,7 @@-#ifndef _COSMO_SOURCE-#define _COSMO_SOURCE-#endif-#ifdef __COSMOPOLITAN__+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "ggml.h"#include <cosmo.h>-#endif#include <errno.h>#include <fcntl.h>#include <stdio.h>@@ -13,7 +11,6 @@#include <limits.h>#include <unistd.h>#include <sys/mman.h>-#include "ggml.h"#include "zip.h"#define Min(a, b) ((a) < (b) ? (a) : (b))@@ -80,12 +77,8 @@ struct ggml_file *ggml_file_open(const char * fname, const char * mode) {}// try opening from this executable's zip store-#ifdef __COSMOPOLITAN__const char *prog = GetProgramExecutableName();-#else- const char *prog = program_invocation_name;-#endif- if ((fd = open(prog, O_RDONLY)) == -1) {+ if ((fd = open(prog, O_RDONLY | O_CLOEXEC)) == -1) {errno = ENOENT;goto Failure;}diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 5800294d..992b386d 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -16958,7 +16958,7 @@ struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context **// read file into data{- FILE * fin = fopen(fname, "rb");+ FILE * fin = fopen(fname, "rbe");if (!fin) {fprintf(stderr, "%s: failed to open %s\n", __func__, fname);return result;diff --git a/llama.cpp/llama.cc b/llama.cpp/llama.ccindex 21ef2316..fcd819a4 100644--- a/llama.cpp/llama.cc+++ b/llama.cpp/llama.cc@@ -1697,7 +1697,7 @@ struct llama_model_loader {struct gguf_context * ctx_gguf = NULL;struct ggml_context * ctx_meta = NULL;- llama_model_loader(const std::string & fname, bool use_mmap) : file(fname.c_str(), "rb") {+ llama_model_loader(const std::string & fname, bool use_mmap) : file(fname.c_str(), "rbe") {struct gguf_init_params params = {/*.no_alloc = */ true,/*.ctx = */ &ctx_meta,@@ -8797,7 +8797,7 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {}static bool llama_load_session_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {- llama_file file(path_session, "rb");+ llama_file file(path_session, "rbe");// sanity checks{diff --git a/llama.cpp/llava/clip.cc b/llama.cpp/llava/clip.ccindex d17e1bf5..8c73373d 100644--- a/llama.cpp/llava/clip.cc+++ b/llama.cpp/llava/clip.cc@@ -463,7 +463,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {struct ggml_file * file;struct gguf_context * ctx;- file = ggml_file_open(fname, "r");+ file = ggml_file_open(fname, "rbe");if (file) ctx = gguf_init_from_file(file, params);if (file) ggml_file_close(file);if (!file || !ctx) {@@ -571,7 +571,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {return nullptr;}- struct ggml_file * fin = ggml_file_open(fname, "r");+ struct ggml_file * fin = ggml_file_open(fname, "rbe");if (!fin) {printf("cannot open model file for loading tensors\n");clip_free(new_clip);diff --git a/llama.cpp/llava/llava.cc b/llama.cpp/llava/llava.ccindex 98887737..c1d87256 100644--- a/llama.cpp/llava/llava.cc+++ b/llama.cpp/llava/llava.cc@@ -110,7 +110,7 @@ LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct cl}static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long *sizeOut) {- auto file = fopen(path, "rb");+ auto file = fopen(path, "rbe");if (file == NULL) {fprintf(stderr, "%s: can't read file %s\n", __func__, path);return false;diff --git a/llama.cpp/main/main.cc b/llama.cpp/main/main.ccindex 70d6c41d..9e0e3e8c 100644--- a/llama.cpp/main/main.cc+++ b/llama.cpp/main/main.cc@@ -225,7 +225,7 @@ int main(int argc, char ** argv) {LOG_TEE("%s: attempting to load saved session from '%s'\n", __func__, path_session.c_str());// fopen to check for existing session- FILE * fp = std::fopen(path_session.c_str(), "rb");+ FILE * fp = std::fopen(path_session.c_str(), "rbe");if (fp != NULL) {std::fclose(fp);diff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mkindex d0a10397..5732cbb2 100644--- a/llama.cpp/server/BUILD.mk+++ b/llama.cpp/server/BUILD.mk@@ -13,6 +13,7 @@ LLAMA_CPP_SERVER_ASSETS := \LLAMA_CPP_SERVER_LINKARGS = \o/$(MODE)/llama.cpp/server/server.o \+ o/$(MODE)/llama.cpp/server/lib/lib.a \o/$(MODE)/llama.cpp/llava/llava.a \o/$(MODE)/llama.cpp/llama.cpp.a@@ -23,6 +24,9 @@ o/$(MODE)/llama.cpp/server/server: \cd llama.cpp/server && zip -q ../../[email protected] $(LLAMA_CPP_SERVER_ASSETS:llama.cpp/server/%=%)mv -f [email protected] $@+include llama.cpp/server/lib/BUILD.mk+.PHONY: o/$(MODE)/llama.cpp/servero/$(MODE)/llama.cpp/server: \- o/$(MODE)/llama.cpp/server/server+ o/$(MODE)/llama.cpp/server/server \+ o/$(MODE)/llama.cpp/server/libdiff --git a/llama.cpp/server/lib/BUILD.mk b/llama.cpp/server/lib/BUILD.mknew file mode 100644index 00000000..c36f2088--- /dev/null+++ b/llama.cpp/server/lib/BUILD.mk@@ -0,0 +1,17 @@+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐+#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘++PKGS += LLAMA_CPP_SERVER_LIB++LLAMA_CPP_SERVER_LIB_FILES := $(wildcard llama.cpp/server/lib/*)+LLAMA_CPP_SERVER_LIB_HDRS = $(filter %.h,$(LLAMA_CPP_SERVER_LIB_FILES))+LLAMA_CPP_SERVER_LIB_SRCS = $(filter %.cc,$(LLAMA_CPP_SERVER_LIB_FILES))+LLAMA_CPP_SERVER_LIB_OBJS = $(LLAMA_CPP_SERVER_LIB_SRCS:%.cc=o/$(MODE)/%.o)++.PHONY: tool/args/args.h++o/$(MODE)/llama.cpp/server/lib/lib.a: $(LLAMA_CPP_SERVER_LIB_OBJS)++.PHONY: o/$(MODE)/llama.cpp/server/lib+o/$(MODE)/llama.cpp/server/lib: \+ o/$(MODE)/llama.cpp/server/lib/lib.adiff --git a/llama.cpp/server/lib/launchbrowser.cc b/llama.cpp/server/lib/launchbrowser.ccnew file mode 100644index 00000000..8fe14fc6--- /dev/null+++ b/llama.cpp/server/lib/launchbrowser.cc@@ -0,0 +1,60 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "llama.cpp/server/lib/lib.h"+#include <cosmo.h>+#include <spawn.h>+#include <errno.h>+#include <stdio.h>+#include <string.h>+#include <sys/wait.h>++static void ReportFailure(const char *url,+ const char *cmd,+ const char *reason) {+ tinyprint(2, "failed to open ", url, " in a browser tab using ", cmd,+ ": ", reason, "\n", NULL);+}++bool LaunchBrowser(const char *url) {++ // determine which command opens browser tab+ const char *cmd;+ if (IsWindows()) {+ cmd = "/c/windows/explorer.exe";+ } else if (IsXnu()) {+ cmd = "open";+ } else {+ cmd = "xdg-open";+ }++ // spawn process+ // set process group so ctrl-c won't kill browser+ int pid, err;+ posix_spawnattr_t sa;+ char *args[] = {(char *)cmd, (char *)url, NULL};+ posix_spawnattr_init(&sa);+ posix_spawnattr_setflags(&sa, POSIX_SPAWN_SETPGROUP);+ err = posix_spawnp(&pid, cmd, NULL, &sa, args, environ);+ posix_spawnattr_destroy(&sa);+ if (err) {+ ReportFailure(url, cmd, strerror(err));+ return false;+ }++ // wait for tab to finish opening+ // the browser will still be running after this completes+ int ws;+ while (waitpid(pid, &ws, 0) == -1) {+ if (errno != EINTR) {+ ReportFailure(url, cmd, strerror(errno));+ return false;+ }+ }+ if (ws) {+ ReportFailure(url, cmd, "process exited with non-zero status");+ return false;+ }++ // report success+ return true;+}diff --git a/llama.cpp/server/lib/lib.h b/llama.cpp/server/lib/lib.hnew file mode 100644index 00000000..c32dbbc0--- /dev/null+++ b/llama.cpp/server/lib/lib.h@@ -0,0 +1,6 @@+#ifndef LLAMA_CPP_SERVER_LIB_H_+#define LLAMA_CPP_SERVER_LIB_H_++bool LaunchBrowser(const char *);++#endif /* LLAMA_CPP_SERVER_LIB_H_ */diff --git a/llama.cpp/server/server.cc b/llama.cpp/server/server.ccindex 923decc2..91d4a88d 100644--- a/llama.cpp/server/server.cc+++ b/llama.cpp/server/server.cc@@ -1,13 +1,13 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi-#ifdef __COSMOPOLITAN__+#include <cosmo.h>#include "tool/args/args.h"-#endif#include "llama.cpp/common.h"#include "llama.cpp/llama.h"#include "llama.cpp/grammar-parser.h"#include "llama.cpp/llava/clip.h"#include "llama.cpp/cpucheck.h"+#include "llama.cpp/server/lib/lib.h"#include "llama.cpp/stb_image.h"#ifndef NDEBUG@@ -18,6 +18,8 @@#include "httplib.h"#include "json.h"+#include <unistd.h>+#include <cstdio>#include <cstddef>#include <thread>#include <mutex>@@ -36,6 +38,7 @@ struct server_paramsint32_t port = 8080;int32_t read_timeout = 600;int32_t write_timeout = 600;+ bool nobrowser;};static bool server_verbose = false;@@ -1805,6 +1808,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" -spf FNAME, --system-prompt-file FNAME\n");printf(" Set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n");printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");+ printf(" --nobrowser Do not attempt to open a web browser tab at startup.\n");printf("\n");}@@ -2159,6 +2163,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}params.mmproj = argv[i];}+ else if (arg == "--nobrowser")+ {+ sparams.nobrowser = true;+ }else{fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());@@ -2528,9 +2536,18 @@ int main(int argc, char **argv)// Set the base directory for serving static filessvr.set_base_dir(sparams.public_path);- // to make it ctrl+clickable:LOG_TEE("\nllama server listening at http://%s:%d\n\n", sparams.hostname.c_str(), sparams.port);+ if (!sparams.nobrowser) {+ char url[128];+ snprintf(url, sizeof(url), "http://%s:%d/",+ sparams.hostname == "0.0.0.0" ? "127.0.0.1" : sparams.hostname.c_str(),+ sparams.port);+ LaunchBrowser(url);+ }++ tinyprint(2, "loading weights...\n", NULL);+LOG_INFO("HTTP server listening", {{"hostname", sparams.hostname},{"port", sparams.port},diff --git a/llama.cpp/stb_image.h b/llama.cpp/stb_image.hindex 4766d7e6..e9825676 100644--- a/llama.cpp/stb_image.h+++ b/llama.cpp/stb_image.h@@ -1325,7 +1325,7 @@ static FILE * stbi__fopen(char const * filename, char const * mode) {}STBIDEF stbi_uc * stbi_load(char const * filename, int * x, int * y, int * comp, int req_comp) {- FILE * f = stbi__fopen(filename, "rb");+ FILE * f = stbi__fopen(filename, "rbe");unsigned char * result;if (!f)return stbi__errpuc("can't fopen", "Unable to open file");@@ -1359,7 +1359,7 @@ STBIDEF stbi__uint16 * stbi_load_from_file_16(FILE * f, int * x, int * y, int *}STBIDEF stbi_us * stbi_load_16(char const * filename, int * x, int * y, int * comp, int req_comp) {- FILE * f = stbi__fopen(filename, "rb");+ FILE * f = stbi__fopen(filename, "rbe");stbi__uint16 * result;if (!f)return (stbi_us *)stbi__errpuc("can't fopen", "Unable to open file");@@ -1447,7 +1447,7 @@ STBIDEF float * stbi_loadf_from_callbacks(stbi_io_callbacks const * clbk, void *#ifndef STBI_NO_STDIOSTBIDEF float * stbi_loadf(char const * filename, int * x, int * y, int * comp, int req_comp) {float * result;- FILE * f = stbi__fopen(filename, "rb");+ FILE * f = stbi__fopen(filename, "rbe");if (!f)return stbi__errpf("can't fopen", "Unable to open file");result = stbi_loadf_from_file(f, x, y, comp, req_comp);@@ -1482,7 +1482,7 @@ STBIDEF int stbi_is_hdr_from_memory(stbi_uc const * buffer, int len) {#ifndef STBI_NO_STDIOSTBIDEF int stbi_is_hdr(char const * filename) {- FILE * f = stbi__fopen(filename, "rb");+ FILE * f = stbi__fopen(filename, "rbe");int result = 0;if (f) {result = stbi_is_hdr_from_file(f);@@ -8107,7 +8107,7 @@ static int stbi__is_16_main(stbi__context * s) {#ifndef STBI_NO_STDIOSTBIDEF int stbi_info(char const * filename, int * x, int * y, int * comp) {- FILE * f = stbi__fopen(filename, "rb");+ FILE * f = stbi__fopen(filename, "rbe");int result;if (!f)return stbi__err("can't fopen", "Unable to open file");@@ -8127,7 +8127,7 @@ STBIDEF int stbi_info_from_file(FILE * f, int * x, int * y, int * comp) {}STBIDEF int stbi_is_16_bit(char const * filename) {- FILE * f = stbi__fopen(filename, "rb");+ FILE * f = stbi__fopen(filename, "rbe");int result;if (!f)return stbi__err("can't fopen", "Unable to open file");diff --git a/llama.cpp/train.cpp b/llama.cpp/train.cppindex bc15b7a0..56d09d85 100644--- a/llama.cpp/train.cpp+++ b/llama.cpp/train.cpp@@ -827,7 +827,7 @@ size_t tokenize_file(std::vector<llama_token> & out_tokens,std::vector<size_t> & out_samples_begin,std::vector<size_t> & out_samples_size) {- struct llama_file f(filename, "rb");+ struct llama_file f(filename, "rbe");if (f.size == 0) {out_tokens.clear();