Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Unify llamafile-server command into llamafile
Justine Tunneyb86dcb769a0a
diff --git a/Makefile b/Makefileindex a3da3d8b..3dfe7bdb 100644--- a/Makefile+++ b/Makefile@@ -29,14 +29,12 @@ install: llamafile/zipalign.1 \llama.cpp/llava/llava-quantize.1 \o/$(MODE)/llamafile/zipalign \o/$(MODE)/llama.cpp/main/main \- o/$(MODE)/llama.cpp/server/server \o/$(MODE)/llama.cpp/quantize/quantize \o/$(MODE)/llama.cpp/perplexity/perplexity \o/$(MODE)/llama.cpp/llava/llava-quantizemkdir -p $(PREFIX)/bin$(INSTALL) o/$(MODE)/llamafile/zipalign $(PREFIX)/bin/zipalign$(INSTALL) o/$(MODE)/llama.cpp/main/main $(PREFIX)/bin/llamafile- $(INSTALL) o/$(MODE)/llama.cpp/server/server $(PREFIX)/bin/llamafile-server$(INSTALL) o/$(MODE)/llama.cpp/quantize/quantize $(PREFIX)/bin/llamafile-quantize$(INSTALL) o/$(MODE)/llama.cpp/perplexity/perplexity $(PREFIX)/bin/llamafile-perplexity$(INSTALL) o/$(MODE)/llama.cpp/llava/llava-quantize $(PREFIX)/bin/llava-quantizediff --git a/README.md b/README.mdindex ed835c00..486fb65b 100644--- a/README.md+++ b/README.md@@ -356,7 +356,7 @@ example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp'srecently-added support for image inputs.```sh-llamafile-server \+llamafile \-m llava-v1.5-7b-Q8_0.gguf \--mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \--host 0.0.0.0@@ -369,7 +369,7 @@ images to it.If you want to be able to just say:```sh-./llava-server.llamafile+./llava.llamafile```...and have it run the web server without having to specify arguments,@@ -393,15 +393,15 @@ user are to be inserted. Next, we'll add both the weights and theargument file to the executable:```sh-cp /usr/local/bin/llamafile-server llava-server.llamafile+cp /usr/local/bin/llamafile llava.llamafilezipalign -j0 \- llava-server.llamafile \+ llava.llamafile \llava-v1.5-7b-Q8_0.gguf \llava-v1.5-7b-mmproj-Q8_0.gguf \.args-./llava-server.llamafile+./llava.llamafile```Congratulations. You've just made your own LLM executable that's easy todiff --git a/llama.cpp/llava/BUILD.mk b/llama.cpp/llava/BUILD.mkindex 9c68e309..0451bcf2 100644--- a/llama.cpp/llava/BUILD.mk+++ b/llama.cpp/llava/BUILD.mk@@ -6,12 +6,12 @@ PKGS += LLAMA_CPP_LLAVALLAMA_CPP_LLAVA_FILES := $(wildcard llama.cpp/llava/*)LLAMA_CPP_LLAVA_HDRS = $(filter %.h,$(LLAMA_CPP_LLAVA_FILES))LLAMA_CPP_LLAVA_SRCS = $(filter %.cpp,$(LLAMA_CPP_LLAVA_FILES))+LLAMA_CPP_LLAVA_OBJS = $(LLAMA_CPP_LLAVA_SRCS:%.cpp=o/$(MODE)/%.o).PHONY: tool/args/args.ho/$(MODE)/llama.cpp/llava/llava.a: \- o/$(MODE)/llama.cpp/llava/clip.o \- o/$(MODE)/llama.cpp/llava/llava.o+ $(LLAMA_CPP_LLAVA_OBJS)o/$(MODE)/llama.cpp/llava/llava-quantize: \o/$(MODE)/llama.cpp/llava/llava-quantize.o \diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mkindex 5e4546dc..64a53dde 100644--- a/llama.cpp/main/BUILD.mk+++ b/llama.cpp/main/BUILD.mk@@ -10,9 +10,10 @@ LLAMA_CPP_MAIN_OBJS = $(LLAMA_CPP_MAIN_SRCS:%.cpp=o/$(MODE)/%.o)o/$(MODE)/llama.cpp/main/main: \o/$(MODE)/llama.cpp/main/main.o \- o/$(MODE)/llama.cpp/llava/llava-cli.o \+ o/$(MODE)/llama.cpp/server/server.a \o/$(MODE)/llama.cpp/llava/llava.a \- o/$(MODE)/llama.cpp/llama.cpp.a+ o/$(MODE)/llama.cpp/llama.cpp.a \+ $(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o).PHONY: o/$(MODE)/llama.cpp/maino/$(MODE)/llama.cpp/main: \diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index 5b1de5b3..ef371543 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -11,14 +11,18 @@.Fl p Ar prompt.Nm.Op flags...+.Op Fl Fl server+.Fl m Ar model.gguf+.Op Fl Fl mmproj Ar vision.gguf+.Nm+.Op flags....Fl m Ar model.gguf.Fl Fl mmproj Ar vision.gguf.Fl Fl image Ar graphic.png.Fl p Ar prompt.Sh DESCRIPTION.Nm-is a command-line tool for running large language models. It has use-cases such as:+is a large language model tool. It has use cases such as:.Pp.Bl -dash -compact.It@@ -37,6 +41,18 @@ The following options are available:Show help message and exit..It Fl Fl versionPrint llamafile version and exit.+.It Fl Fl server+.Xr llamafile-server 1+is launched via the+.Nm llamafile+command when the+.Fl Fl server+flag is passed. The server flag is implied by default if no prompt is+specified, i.e. neither the+.Fl p+or+.Fl f+flags are passed..It Fl m Ar FNAME , Fl Fl model Ar FNAMEModel path in GGUF file format..Ppdiff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 6490ffb3..641b318b 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -24,6 +24,7 @@#include "llama.cpp/ggml-cuda.h"#include "llama.cpp/ggml-metal.h"#include "llama.cpp/llava/llava.h"+#include "llama.cpp/server/server.h"static llama_context ** g_ctx;static llama_model ** g_model;@@ -108,11 +109,18 @@ static bool has_argument(int argc, char ** argv, const char * arg) {}int main(int argc, char ** argv) {- if (argc == 2 && !strcmp(argv[1], "--version")) {- printf("llamafile v" LLAMAFILE_VERSION_STRING " main\n");++ if (has_argument(argc, argv, "--version")) {+ printf("llamafile v" LLAMAFILE_VERSION_STRING "\n");exit(0);}+ if (has_argument(argc, argv, "--server") ||+ (!has_argument(argc, argv, "-p") &&+ !has_argument(argc, argv, "-f"))) {+ return server_cli(argc, argv);+ }+llamafile_init();llamafile_check_cpu();ShowCrashReports();diff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mkindex d1549a1d..15173797 100644--- a/llama.cpp/server/BUILD.mk+++ b/llama.cpp/server/BUILD.mk@@ -7,15 +7,13 @@ LLAMA_CPP_SERVER_FILES := $(wildcard llama.cpp/server/*)LLAMA_CPP_SERVER_ASSETS := $(wildcard llama.cpp/server/public/*)LLAMA_CPP_SERVER_HDRS = $(filter %.h,$(LLAMA_CPP_SERVER_FILES))LLAMA_CPP_SERVER_SRCS = $(filter %.cpp,$(LLAMA_CPP_SERVER_FILES))+LLAMA_CPP_SERVER_OBJS = $(LLAMA_CPP_SERVER_SRCS:%.cpp=o/$(MODE)/%.o)-o/$(MODE)/llama.cpp/server/server.o: private CXXFLAGS += -O1+o/$(MODE)/llama.cpp/server/impl.o: private CXXFLAGS += -O1-o/$(MODE)/llama.cpp/server/server: \- o/$(MODE)/llama.cpp/server/server.o \- o/$(MODE)/llama.cpp/llava/llava.a \- o/$(MODE)/llama.cpp/llama.cpp.a \- $(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)+o/$(MODE)/llama.cpp/server/server.a: \+ $(LLAMA_CPP_SERVER_OBJS).PHONY: o/$(MODE)/llama.cpp/servero/$(MODE)/llama.cpp/server: \- o/$(MODE)/llama.cpp/server/server+ o/$(MODE)/llama.cpp/server/server.adiff --git a/llama.cpp/server/server.1 b/llama.cpp/server/server.1index 90e87def..69d18ca7 100644--- a/llama.cpp/server/server.1+++ b/llama.cpp/server/server.1@@ -5,7 +5,8 @@.Nm llamafile-server.Nd large language model server.Sh SYNOPSIS-.Nm+.Nm llamafile+.Fl Fl server.Op flags....Op Fl m Ar weights.gguf.Sh DESCRIPTION@@ -14,6 +15,18 @@ launches an HTTP server and browser tab with a web user interface forthe llama.cpp software. The UI lets you chat with your LLM and uploadimages it can analyze. Furthermore, this server provides an OpenAI APIcompatible completions endpoint.+.Pp+.Nm llamafile-server+is launched via the+.Nm llamafile+command when the+.Fl Fl server+flag is passed. The server flag is implied by default if no prompt is+specified, i.e. neither the+.Fl p+or+.Fl f+flags are passed..Sh OPTIONSThe following options are available:.Bl -tag -width indentdiff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 882c83d2..d5c80beb 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -1,5 +1,7 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi++#include "server.h"#include "tool/args/args.h"#include "llama.cpp/common.h"#include "llama.cpp/llama.h"@@ -2323,6 +2325,9 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,{sparams.unsecure = true;}+ else if (arg == "--server")+ {+ }else{fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());@@ -2645,7 +2650,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con}}-int main(int argc, char **argv)+int server_cli(int argc, char ** argv){if (argc == 2 && !strcmp(argv[1], "--version")) {printf("llamafile v" LLAMAFILE_VERSION_STRING " server\n");diff --git a/llama.cpp/server/server.h b/llama.cpp/server/server.hnew file mode 100644index 00000000..f90fa4e2--- /dev/null+++ b/llama.cpp/server/server.h@@ -0,0 +1,2 @@+#pragma once+int server_cli(int, char **);