Skip to content

Distribute and run LLMs with a single file.

Unify llamafile-server command into llamafile

Justine Tunneyb86dcb769a0a
diff --git a/Makefile b/Makefile
index a3da3d8b..3dfe7bdb 100644
--- a/Makefile
+++ b/Makefile
@@ -29,14 +29,12 @@ install: llamafile/zipalign.1 \
llama.cpp/llava/llava-quantize.1 \
o/$(MODE)/llamafile/zipalign \
o/$(MODE)/llama.cpp/main/main \
- o/$(MODE)/llama.cpp/server/server \
o/$(MODE)/llama.cpp/quantize/quantize \
o/$(MODE)/llama.cpp/perplexity/perplexity \
o/$(MODE)/llama.cpp/llava/llava-quantize
mkdir -p $(PREFIX)/bin
$(INSTALL) o/$(MODE)/llamafile/zipalign $(PREFIX)/bin/zipalign
$(INSTALL) o/$(MODE)/llama.cpp/main/main $(PREFIX)/bin/llamafile
- $(INSTALL) o/$(MODE)/llama.cpp/server/server $(PREFIX)/bin/llamafile-server
$(INSTALL) o/$(MODE)/llama.cpp/quantize/quantize $(PREFIX)/bin/llamafile-quantize
$(INSTALL) o/$(MODE)/llama.cpp/perplexity/perplexity $(PREFIX)/bin/llamafile-perplexity
$(INSTALL) o/$(MODE)/llama.cpp/llava/llava-quantize $(PREFIX)/bin/llava-quantize
diff --git a/README.md b/README.md
index ed835c00..486fb65b 100644
--- a/README.md
+++ b/README.md
@@ -356,7 +356,7 @@ example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's
recently-added support for image inputs.
```sh
-llamafile-server \
+llamafile \
-m llava-v1.5-7b-Q8_0.gguf \
--mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \
--host 0.0.0.0
@@ -369,7 +369,7 @@ images to it.
If you want to be able to just say:
```sh
-./llava-server.llamafile
+./llava.llamafile
```
...and have it run the web server without having to specify arguments,
@@ -393,15 +393,15 @@ user are to be inserted. Next, we'll add both the weights and the
argument file to the executable:
```sh
-cp /usr/local/bin/llamafile-server llava-server.llamafile
+cp /usr/local/bin/llamafile llava.llamafile
zipalign -j0 \
- llava-server.llamafile \
+ llava.llamafile \
llava-v1.5-7b-Q8_0.gguf \
llava-v1.5-7b-mmproj-Q8_0.gguf \
.args
-./llava-server.llamafile
+./llava.llamafile
```
Congratulations. You've just made your own LLM executable that's easy to
diff --git a/llama.cpp/llava/BUILD.mk b/llama.cpp/llava/BUILD.mk
index 9c68e309..0451bcf2 100644
--- a/llama.cpp/llava/BUILD.mk
+++ b/llama.cpp/llava/BUILD.mk
@@ -6,12 +6,12 @@ PKGS += LLAMA_CPP_LLAVA
LLAMA_CPP_LLAVA_FILES := $(wildcard llama.cpp/llava/*)
LLAMA_CPP_LLAVA_HDRS = $(filter %.h,$(LLAMA_CPP_LLAVA_FILES))
LLAMA_CPP_LLAVA_SRCS = $(filter %.cpp,$(LLAMA_CPP_LLAVA_FILES))
+LLAMA_CPP_LLAVA_OBJS = $(LLAMA_CPP_LLAVA_SRCS:%.cpp=o/$(MODE)/%.o)
.PHONY: tool/args/args.h
o/$(MODE)/llama.cpp/llava/llava.a: \
- o/$(MODE)/llama.cpp/llava/clip.o \
- o/$(MODE)/llama.cpp/llava/llava.o
+ $(LLAMA_CPP_LLAVA_OBJS)
o/$(MODE)/llama.cpp/llava/llava-quantize: \
o/$(MODE)/llama.cpp/llava/llava-quantize.o \
diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mk
index 5e4546dc..64a53dde 100644
--- a/llama.cpp/main/BUILD.mk
+++ b/llama.cpp/main/BUILD.mk
@@ -10,9 +10,10 @@ LLAMA_CPP_MAIN_OBJS = $(LLAMA_CPP_MAIN_SRCS:%.cpp=o/$(MODE)/%.o)
o/$(MODE)/llama.cpp/main/main: \
o/$(MODE)/llama.cpp/main/main.o \
- o/$(MODE)/llama.cpp/llava/llava-cli.o \
+ o/$(MODE)/llama.cpp/server/server.a \
o/$(MODE)/llama.cpp/llava/llava.a \
- o/$(MODE)/llama.cpp/llama.cpp.a
+ o/$(MODE)/llama.cpp/llama.cpp.a \
+ $(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
.PHONY: o/$(MODE)/llama.cpp/main
o/$(MODE)/llama.cpp/main: \
diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1
index 5b1de5b3..ef371543 100644
--- a/llama.cpp/main/main.1
+++ b/llama.cpp/main/main.1
@@ -11,14 +11,18 @@
.Fl p Ar prompt
.Nm
.Op flags...
+.Op Fl Fl server
+.Fl m Ar model.gguf
+.Op Fl Fl mmproj Ar vision.gguf
+.Nm
+.Op flags...
.Fl m Ar model.gguf
.Fl Fl mmproj Ar vision.gguf
.Fl Fl image Ar graphic.png
.Fl p Ar prompt
.Sh DESCRIPTION
.Nm
-is a command-line tool for running large language models. It has use
-cases such as:
+is a large language model tool. It has use cases such as:
.Pp
.Bl -dash -compact
.It
@@ -37,6 +41,18 @@ The following options are available:
Show help message and exit.
.It Fl Fl version
Print llamafile version and exit.
+.It Fl Fl server
+.Xr llamafile-server 1
+is launched via the
+.Nm llamafile
+command when the
+.Fl Fl server
+flag is passed. The server flag is implied by default if no prompt is
+specified, i.e. neither the
+.Fl p
+or
+.Fl f
+flags are passed.
.It Fl m Ar FNAME , Fl Fl model Ar FNAME
Model path in GGUF file format.
.Pp
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 6490ffb3..641b318b 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -24,6 +24,7 @@
#include "llama.cpp/ggml-cuda.h"
#include "llama.cpp/ggml-metal.h"
#include "llama.cpp/llava/llava.h"
+#include "llama.cpp/server/server.h"
static llama_context ** g_ctx;
static llama_model ** g_model;
@@ -108,11 +109,18 @@ static bool has_argument(int argc, char ** argv, const char * arg) {
}
int main(int argc, char ** argv) {
- if (argc == 2 && !strcmp(argv[1], "--version")) {
- printf("llamafile v" LLAMAFILE_VERSION_STRING " main\n");
+
+ if (has_argument(argc, argv, "--version")) {
+ printf("llamafile v" LLAMAFILE_VERSION_STRING "\n");
exit(0);
}
+ if (has_argument(argc, argv, "--server") ||
+ (!has_argument(argc, argv, "-p") &&
+ !has_argument(argc, argv, "-f"))) {
+ return server_cli(argc, argv);
+ }
+
llamafile_init();
llamafile_check_cpu();
ShowCrashReports();
diff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mk
index d1549a1d..15173797 100644
--- a/llama.cpp/server/BUILD.mk
+++ b/llama.cpp/server/BUILD.mk
@@ -7,15 +7,13 @@ LLAMA_CPP_SERVER_FILES := $(wildcard llama.cpp/server/*)
LLAMA_CPP_SERVER_ASSETS := $(wildcard llama.cpp/server/public/*)
LLAMA_CPP_SERVER_HDRS = $(filter %.h,$(LLAMA_CPP_SERVER_FILES))
LLAMA_CPP_SERVER_SRCS = $(filter %.cpp,$(LLAMA_CPP_SERVER_FILES))
+LLAMA_CPP_SERVER_OBJS = $(LLAMA_CPP_SERVER_SRCS:%.cpp=o/$(MODE)/%.o)
-o/$(MODE)/llama.cpp/server/server.o: private CXXFLAGS += -O1
+o/$(MODE)/llama.cpp/server/impl.o: private CXXFLAGS += -O1
-o/$(MODE)/llama.cpp/server/server: \
- o/$(MODE)/llama.cpp/server/server.o \
- o/$(MODE)/llama.cpp/llava/llava.a \
- o/$(MODE)/llama.cpp/llama.cpp.a \
- $(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
+o/$(MODE)/llama.cpp/server/server.a: \
+ $(LLAMA_CPP_SERVER_OBJS)
.PHONY: o/$(MODE)/llama.cpp/server
o/$(MODE)/llama.cpp/server: \
- o/$(MODE)/llama.cpp/server/server
+ o/$(MODE)/llama.cpp/server/server.a
diff --git a/llama.cpp/server/server.1 b/llama.cpp/server/server.1
index 90e87def..69d18ca7 100644
--- a/llama.cpp/server/server.1
+++ b/llama.cpp/server/server.1
@@ -5,7 +5,8 @@
.Nm llamafile-server
.Nd large language model server
.Sh SYNOPSIS
-.Nm
+.Nm llamafile
+.Fl Fl server
.Op flags...
.Op Fl m Ar weights.gguf
.Sh DESCRIPTION
@@ -14,6 +15,18 @@ launches an HTTP server and browser tab with a web user interface for
the llama.cpp software. The UI lets you chat with your LLM and upload
images it can analyze. Furthermore, this server provides an OpenAI API
compatible completions endpoint.
+.Pp
+.Nm llamafile-server
+is launched via the
+.Nm llamafile
+command when the
+.Fl Fl server
+flag is passed. The server flag is implied by default if no prompt is
+specified, i.e. neither the
+.Fl p
+or
+.Fl f
+flags are passed.
.Sh OPTIONS
The following options are available:
.Bl -tag -width indent
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index 882c83d2..d5c80beb 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -1,5 +1,7 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+
+#include "server.h"
#include "tool/args/args.h"
#include "llama.cpp/common.h"
#include "llama.cpp/llama.h"
@@ -2323,6 +2325,9 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
{
sparams.unsecure = true;
}
+ else if (arg == "--server")
+ {
+ }
else
{
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
@@ -2645,7 +2650,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
}
}
-int main(int argc, char **argv)
+int server_cli(int argc, char ** argv)
{
if (argc == 2 && !strcmp(argv[1], "--version")) {
printf("llamafile v" LLAMAFILE_VERSION_STRING " server\n");
diff --git a/llama.cpp/server/server.h b/llama.cpp/server/server.h
new file mode 100644
index 00000000..f90fa4e2
--- /dev/null
+++ b/llama.cpp/server/server.h
@@ -0,0 +1,2 @@
+#pragma once
+int server_cli(int, char **);