Skip to content

Distribute and run LLMs with a single file.

Modify server.cpp to open browser tab on startup

Justine Tunney042ba195be5a
diff --git a/build/config.mk b/build/config.mk
index 0f6448b7..c500d103 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -23,6 +23,7 @@ endif
LC_ALL = C
SOURCE_DATE_EPOCH = 0
+export MODE
export TMPDIR
export LC_ALL
export SOURCE_DATE_EPOCH
diff --git a/llama.cpp/ggml-file.c b/llama.cpp/ggml-file.c
index 6e0c86b4..7b33bd65 100644
--- a/llama.cpp/ggml-file.c
+++ b/llama.cpp/ggml-file.c
@@ -1,9 +1,7 @@
-#ifndef _COSMO_SOURCE
-#define _COSMO_SOURCE
-#endif
-#ifdef __COSMOPOLITAN__
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#include "ggml.h"
#include <cosmo.h>
-#endif
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
@@ -13,7 +11,6 @@
#include <limits.h>
#include <unistd.h>
#include <sys/mman.h>
-#include "ggml.h"
#include "zip.h"
#define Min(a, b) ((a) < (b) ? (a) : (b))
@@ -80,12 +77,8 @@ struct ggml_file *ggml_file_open(const char * fname, const char * mode) {
}
// try opening from this executable's zip store
-#ifdef __COSMOPOLITAN__
const char *prog = GetProgramExecutableName();
-#else
- const char *prog = program_invocation_name;
-#endif
- if ((fd = open(prog, O_RDONLY)) == -1) {
+ if ((fd = open(prog, O_RDONLY | O_CLOEXEC)) == -1) {
errno = ENOENT;
goto Failure;
}
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 5800294d..992b386d 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -16958,7 +16958,7 @@ struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context **
// read file into data
{
- FILE * fin = fopen(fname, "rb");
+ FILE * fin = fopen(fname, "rbe");
if (!fin) {
fprintf(stderr, "%s: failed to open %s\n", __func__, fname);
return result;
diff --git a/llama.cpp/llama.cc b/llama.cpp/llama.cc
index 21ef2316..fcd819a4 100644
--- a/llama.cpp/llama.cc
+++ b/llama.cpp/llama.cc
@@ -1697,7 +1697,7 @@ struct llama_model_loader {
struct gguf_context * ctx_gguf = NULL;
struct ggml_context * ctx_meta = NULL;
- llama_model_loader(const std::string & fname, bool use_mmap) : file(fname.c_str(), "rb") {
+ llama_model_loader(const std::string & fname, bool use_mmap) : file(fname.c_str(), "rbe") {
struct gguf_init_params params = {
/*.no_alloc = */ true,
/*.ctx = */ &ctx_meta,
@@ -8797,7 +8797,7 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {
}
static bool llama_load_session_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
- llama_file file(path_session, "rb");
+ llama_file file(path_session, "rbe");
// sanity checks
{
diff --git a/llama.cpp/llava/clip.cc b/llama.cpp/llava/clip.cc
index d17e1bf5..8c73373d 100644
--- a/llama.cpp/llava/clip.cc
+++ b/llama.cpp/llava/clip.cc
@@ -463,7 +463,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
struct ggml_file * file;
struct gguf_context * ctx;
- file = ggml_file_open(fname, "r");
+ file = ggml_file_open(fname, "rbe");
if (file) ctx = gguf_init_from_file(file, params);
if (file) ggml_file_close(file);
if (!file || !ctx) {
@@ -571,7 +571,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
return nullptr;
}
- struct ggml_file * fin = ggml_file_open(fname, "r");
+ struct ggml_file * fin = ggml_file_open(fname, "rbe");
if (!fin) {
printf("cannot open model file for loading tensors\n");
clip_free(new_clip);
diff --git a/llama.cpp/llava/llava.cc b/llama.cpp/llava/llava.cc
index 98887737..c1d87256 100644
--- a/llama.cpp/llava/llava.cc
+++ b/llama.cpp/llava/llava.cc
@@ -110,7 +110,7 @@ LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct cl
}
static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long *sizeOut) {
- auto file = fopen(path, "rb");
+ auto file = fopen(path, "rbe");
if (file == NULL) {
fprintf(stderr, "%s: can't read file %s\n", __func__, path);
return false;
diff --git a/llama.cpp/main/main.cc b/llama.cpp/main/main.cc
index 70d6c41d..9e0e3e8c 100644
--- a/llama.cpp/main/main.cc
+++ b/llama.cpp/main/main.cc
@@ -225,7 +225,7 @@ int main(int argc, char ** argv) {
LOG_TEE("%s: attempting to load saved session from '%s'\n", __func__, path_session.c_str());
// fopen to check for existing session
- FILE * fp = std::fopen(path_session.c_str(), "rb");
+ FILE * fp = std::fopen(path_session.c_str(), "rbe");
if (fp != NULL) {
std::fclose(fp);
diff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mk
index d0a10397..5732cbb2 100644
--- a/llama.cpp/server/BUILD.mk
+++ b/llama.cpp/server/BUILD.mk
@@ -13,6 +13,7 @@ LLAMA_CPP_SERVER_ASSETS := \
LLAMA_CPP_SERVER_LINKARGS = \
o/$(MODE)/llama.cpp/server/server.o \
+ o/$(MODE)/llama.cpp/server/lib/lib.a \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a
@@ -23,6 +24,9 @@ o/$(MODE)/llama.cpp/server/server: \
cd llama.cpp/server && zip -q ../../[email protected] $(LLAMA_CPP_SERVER_ASSETS:llama.cpp/server/%=%)
+include llama.cpp/server/lib/BUILD.mk
+
.PHONY: o/$(MODE)/llama.cpp/server
o/$(MODE)/llama.cpp/server: \
- o/$(MODE)/llama.cpp/server/server
+ o/$(MODE)/llama.cpp/server/server \
+ o/$(MODE)/llama.cpp/server/lib
diff --git a/llama.cpp/server/lib/BUILD.mk b/llama.cpp/server/lib/BUILD.mk
new file mode 100644
index 00000000..c36f2088
--- /dev/null
+++ b/llama.cpp/server/lib/BUILD.mk
@@ -0,0 +1,17 @@
+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐
+#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘
+
+PKGS += LLAMA_CPP_SERVER_LIB
+
+LLAMA_CPP_SERVER_LIB_FILES := $(wildcard llama.cpp/server/lib/*)
+LLAMA_CPP_SERVER_LIB_HDRS = $(filter %.h,$(LLAMA_CPP_SERVER_LIB_FILES))
+LLAMA_CPP_SERVER_LIB_SRCS = $(filter %.cc,$(LLAMA_CPP_SERVER_LIB_FILES))
+LLAMA_CPP_SERVER_LIB_OBJS = $(LLAMA_CPP_SERVER_LIB_SRCS:%.cc=o/$(MODE)/%.o)
+
+.PHONY: tool/args/args.h
+
+o/$(MODE)/llama.cpp/server/lib/lib.a: $(LLAMA_CPP_SERVER_LIB_OBJS)
+
+.PHONY: o/$(MODE)/llama.cpp/server/lib
+o/$(MODE)/llama.cpp/server/lib: \
+ o/$(MODE)/llama.cpp/server/lib/lib.a
diff --git a/llama.cpp/server/lib/launchbrowser.cc b/llama.cpp/server/lib/launchbrowser.cc
new file mode 100644
index 00000000..8fe14fc6
--- /dev/null
+++ b/llama.cpp/server/lib/launchbrowser.cc
@@ -0,0 +1,60 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#include "llama.cpp/server/lib/lib.h"
+#include <cosmo.h>
+#include <spawn.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/wait.h>
+
+static void ReportFailure(const char *url,
+ const char *cmd,
+ const char *reason) {
+ tinyprint(2, "failed to open ", url, " in a browser tab using ", cmd,
+ ": ", reason, "\n", NULL);
+}
+
+bool LaunchBrowser(const char *url) {
+
+ // determine which command opens browser tab
+ const char *cmd;
+ if (IsWindows()) {
+ cmd = "/c/windows/explorer.exe";
+ } else if (IsXnu()) {
+ cmd = "open";
+ } else {
+ cmd = "xdg-open";
+ }
+
+ // spawn process
+ // set process group so ctrl-c won't kill browser
+ int pid, err;
+ posix_spawnattr_t sa;
+ char *args[] = {(char *)cmd, (char *)url, NULL};
+ posix_spawnattr_init(&sa);
+ posix_spawnattr_setflags(&sa, POSIX_SPAWN_SETPGROUP);
+ err = posix_spawnp(&pid, cmd, NULL, &sa, args, environ);
+ posix_spawnattr_destroy(&sa);
+ if (err) {
+ ReportFailure(url, cmd, strerror(err));
+ return false;
+ }
+
+ // wait for tab to finish opening
+ // the browser will still be running after this completes
+ int ws;
+ while (waitpid(pid, &ws, 0) == -1) {
+ if (errno != EINTR) {
+ ReportFailure(url, cmd, strerror(errno));
+ return false;
+ }
+ }
+ if (ws) {
+ ReportFailure(url, cmd, "process exited with non-zero status");
+ return false;
+ }
+
+ // report success
+ return true;
+}
diff --git a/llama.cpp/server/lib/lib.h b/llama.cpp/server/lib/lib.h
new file mode 100644
index 00000000..c32dbbc0
--- /dev/null
+++ b/llama.cpp/server/lib/lib.h
@@ -0,0 +1,6 @@
+#ifndef LLAMA_CPP_SERVER_LIB_H_
+#define LLAMA_CPP_SERVER_LIB_H_
+
+bool LaunchBrowser(const char *);
+
+#endif /* LLAMA_CPP_SERVER_LIB_H_ */
diff --git a/llama.cpp/server/server.cc b/llama.cpp/server/server.cc
index 923decc2..91d4a88d 100644
--- a/llama.cpp/server/server.cc
+++ b/llama.cpp/server/server.cc
@@ -1,13 +1,13 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-
// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#ifdef __COSMOPOLITAN__
+#include <cosmo.h>
#include "tool/args/args.h"
-#endif
#include "llama.cpp/common.h"
#include "llama.cpp/llama.h"
#include "llama.cpp/grammar-parser.h"
#include "llama.cpp/llava/clip.h"
#include "llama.cpp/cpucheck.h"
+#include "llama.cpp/server/lib/lib.h"
#include "llama.cpp/stb_image.h"
#ifndef NDEBUG
@@ -18,6 +18,8 @@
#include "httplib.h"
#include "json.h"
+#include <unistd.h>
+#include <cstdio>
#include <cstddef>
#include <thread>
#include <mutex>
@@ -36,6 +38,7 @@ struct server_params
int32_t port = 8080;
int32_t read_timeout = 600;
int32_t write_timeout = 600;
+ bool nobrowser;
};
static bool server_verbose = false;
@@ -1805,6 +1808,7 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
printf(" -spf FNAME, --system-prompt-file FNAME\n");
printf(" Set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n");
printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");
+ printf(" --nobrowser Do not attempt to open a web browser tab at startup.\n");
printf("\n");
}
@@ -2159,6 +2163,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
}
params.mmproj = argv[i];
}
+ else if (arg == "--nobrowser")
+ {
+ sparams.nobrowser = true;
+ }
else
{
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
@@ -2528,9 +2536,18 @@ int main(int argc, char **argv)
// Set the base directory for serving static files
svr.set_base_dir(sparams.public_path);
- // to make it ctrl+clickable:
LOG_TEE("\nllama server listening at http://%s:%d\n\n", sparams.hostname.c_str(), sparams.port);
+ if (!sparams.nobrowser) {
+ char url[128];
+ snprintf(url, sizeof(url), "http://%s:%d/",
+ sparams.hostname == "0.0.0.0" ? "127.0.0.1" : sparams.hostname.c_str(),
+ sparams.port);
+ LaunchBrowser(url);
+ }
+
+ tinyprint(2, "loading weights...\n", NULL);
+
LOG_INFO("HTTP server listening", {
{"hostname", sparams.hostname},
{"port", sparams.port},
diff --git a/llama.cpp/stb_image.h b/llama.cpp/stb_image.h
index 4766d7e6..e9825676 100644
--- a/llama.cpp/stb_image.h
+++ b/llama.cpp/stb_image.h
@@ -1325,7 +1325,7 @@ static FILE * stbi__fopen(char const * filename, char const * mode) {
}
STBIDEF stbi_uc * stbi_load(char const * filename, int * x, int * y, int * comp, int req_comp) {
- FILE * f = stbi__fopen(filename, "rb");
+ FILE * f = stbi__fopen(filename, "rbe");
unsigned char * result;
if (!f)
return stbi__errpuc("can't fopen", "Unable to open file");
@@ -1359,7 +1359,7 @@ STBIDEF stbi__uint16 * stbi_load_from_file_16(FILE * f, int * x, int * y, int *
}
STBIDEF stbi_us * stbi_load_16(char const * filename, int * x, int * y, int * comp, int req_comp) {
- FILE * f = stbi__fopen(filename, "rb");
+ FILE * f = stbi__fopen(filename, "rbe");
stbi__uint16 * result;
if (!f)
return (stbi_us *)stbi__errpuc("can't fopen", "Unable to open file");
@@ -1447,7 +1447,7 @@ STBIDEF float * stbi_loadf_from_callbacks(stbi_io_callbacks const * clbk, void *
#ifndef STBI_NO_STDIO
STBIDEF float * stbi_loadf(char const * filename, int * x, int * y, int * comp, int req_comp) {
float * result;
- FILE * f = stbi__fopen(filename, "rb");
+ FILE * f = stbi__fopen(filename, "rbe");
if (!f)
return stbi__errpf("can't fopen", "Unable to open file");
result = stbi_loadf_from_file(f, x, y, comp, req_comp);
@@ -1482,7 +1482,7 @@ STBIDEF int stbi_is_hdr_from_memory(stbi_uc const * buffer, int len) {
#ifndef STBI_NO_STDIO
STBIDEF int stbi_is_hdr(char const * filename) {
- FILE * f = stbi__fopen(filename, "rb");
+ FILE * f = stbi__fopen(filename, "rbe");
int result = 0;
if (f) {
result = stbi_is_hdr_from_file(f);
@@ -8107,7 +8107,7 @@ static int stbi__is_16_main(stbi__context * s) {
#ifndef STBI_NO_STDIO
STBIDEF int stbi_info(char const * filename, int * x, int * y, int * comp) {
- FILE * f = stbi__fopen(filename, "rb");
+ FILE * f = stbi__fopen(filename, "rbe");
int result;
if (!f)
return stbi__err("can't fopen", "Unable to open file");
@@ -8127,7 +8127,7 @@ STBIDEF int stbi_info_from_file(FILE * f, int * x, int * y, int * comp) {
}
STBIDEF int stbi_is_16_bit(char const * filename) {
- FILE * f = stbi__fopen(filename, "rb");
+ FILE * f = stbi__fopen(filename, "rbe");
int result;
if (!f)
return stbi__err("can't fopen", "Unable to open file");
diff --git a/llama.cpp/train.cpp b/llama.cpp/train.cpp
index bc15b7a0..56d09d85 100644
--- a/llama.cpp/train.cpp
+++ b/llama.cpp/train.cpp
@@ -827,7 +827,7 @@ size_t tokenize_file(
std::vector<llama_token> & out_tokens,
std::vector<size_t> & out_samples_begin,
std::vector<size_t> & out_samples_size) {
- struct llama_file f(filename, "rb");
+ struct llama_file f(filename, "rbe");
if (f.size == 0) {
out_tokens.clear();