Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Introduce new llamafile server
Justine Tunneye0656ea190fa
diff --git a/build/config.mk b/build/config.mkindex a7f25520..0bf1af6b 100644--- a/build/config.mk+++ b/build/config.mk@@ -2,7 +2,7 @@#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘PREFIX = /usr/local-COSMOCC = .cosmocc/3.3.10+COSMOCC = .cosmocc/3.4.0TOOLCHAIN = $(COSMOCC)/bin/cosmoAR = $(TOOLCHAIN)ar@@ -13,7 +13,8 @@ MKDEPS = $(COSMOCC)/bin/mkdepsINSTALL = installARFLAGS = rcsD-CCFLAGS = -g -O3 -fexceptions -fsignaling-nans+CXXFLAGS = -frtti -std=gnu++23+CCFLAGS = -g -ggdb -O3 -fexceptions -fsignaling-nans -ffunction-sections -fdata-sectionsCPPFLAGS_ = -iquote. -mcosmo -DGGML_MULTIPLATFORM -Wno-attributes -DLLAMAFILE_DEBUGTARGET_ARCH = -Xx86_64-mavx -Xx86_64-mtune=znver4@@ -51,5 +52,5 @@ clean:; rm -rf o.PHONY: distcleandistclean:; rm -rf o .cosmocc-.cosmocc/3.3.10:- build/download-cosmocc.sh $@ 3.3.10 00d61c1215667314f66e288c8285bae38cc6137fca083e5bba6c74e3a52439de+.cosmocc/3.4.0:+ build/download-cosmocc.sh $@ 3.4.0 475e24b84a18973312433f5280e267acbe1b4dac1b2e2ebb3cfce46051a8c08cdiff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 30f4eab0..2be033db 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -12,11 +12,17 @@ LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU)LLAMAFILE_DOCS = $(filter %.1,$(LLAMAFILE_FILES))-LLAMAFILE_OBJS = \+LLAMAFILE_OBJS := \$(LLAMAFILE_SRCS_C:%.c=o/$(MODE)/%.o) \$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)+# this executable defines its own malloc(), free(), etc.+# therefore we want to avoid it going inside the .a file+LLAMAFILE_OBJS := $(filter-out o/$(MODE)/llamafile/zipalign.o,$(LLAMAFILE_OBJS))++include llamafile/server/BUILD.mk+o/$(MODE)/llamafile/zipalign: \o/$(MODE)/llamafile/zipalign.o \o/$(MODE)/llamafile/help.o \@@ -29,7 +35,6 @@ o/$(MODE)/llamafile/zipcheck: \o/$(MODE)/llamafile/simple: \o/$(MODE)/llamafile/simple.o \- o/$(MODE)/llama.cpp/llava/llava.a \o/$(MODE)/llama.cpp/llama.cpp.ao/$(MODE)/llamafile/tokenize: \@@ -39,6 +44,7 @@ o/$(MODE)/llamafile/tokenize: \.PHONY: o/$(MODE)/llamafileo/$(MODE)/llamafile: \$(LLAMAFILE_OBJS) \+ o/$(MODE)/llamafile/server \o/$(MODE)/llamafile/simple \o/$(MODE)/llamafile/zipalign \o/$(MODE)/llamafile/zipcheck \diff --git a/llamafile/debug.cpp b/llamafile/debug.cppindex d44177f4..eb3acd69 100644--- a/llamafile/debug.cpp+++ b/llamafile/debug.cpp@@ -18,12 +18,12 @@#include "debug.h"#include "log.h"+#include <atomic>#include <cosmo.h>#include <fenv.h>#include <libc/calls/struct/aarch64.internal.h>#include <libc/calls/struct/ucontext.internal.h>#include <signal.h>-#include <stdatomic.h>#include <termios.h>#include <ucontext.h>#include <unistd.h>@@ -36,7 +36,7 @@#define UNDERFLOW_DELAY 2bool FLAG_trap;-static atomic_llong g_underflowed;+static std::atomic_llong g_underflowed;static thread_local int g_enabled;thread_local int llamafile_debug_op_index;const struct ggml_cgraph *llamafile_debug_graph;@@ -59,17 +59,17 @@ static long long millis(void) {return timespec_tomillis(timespec_real());}-static inline void spinlock(atomic_uint *lock) {+static inline void spinlock(std::atomic_uint *lock) {int x;for (;;) {- x = atomic_exchange_explicit(lock, 1, memory_order_acquire);+ x = lock->exchange(1, std::memory_order_acquire);if (!x)break;}}-static inline void spunlock(atomic_uint *lock) {- atomic_store_explicit(lock, 0, memory_order_release);+static inline void spunlock(std::atomic_uint *lock) {+ lock->store(0, std::memory_order_release);}static const char *describe_vertex(struct ggml_tensor *t) {@@ -130,8 +130,7 @@ static void on_sigfpe(int sig, siginfo_t *si, void *arg) {if (reason == FPE_FLTUND) {if (g_terminal_buddy.is_terminal) {long long now = millis();- if ((now - atomic_exchange_explicit(&g_underflowed, now, memory_order_relaxed)) >- UNDERFLOW_DELAY) {+ if ((now - g_underflowed.exchange(now, std::memory_order_relaxed)) > UNDERFLOW_DELAY) {write(2, UNDERFLOW_ALARM, strlen(UNDERFLOW_ALARM));}}@@ -139,7 +138,7 @@ static void on_sigfpe(int sig, siginfo_t *si, void *arg) {return;}- static atomic_uint lock;+ static std::atomic_uint lock;spinlock(&lock);const char *issue;@@ -205,7 +204,7 @@ static void setup_sigfpe(void) {}int llamafile_trapping_enabled(int delta) {- static atomic_uint once;+ static _Atomic(uint32_t) once;bool was_enabled = g_enabled > 0;bool is_enabled = (g_enabled += delta) > 0;feclearexcept(FE_ALL_EXCEPT);@@ -225,11 +224,10 @@ void llamafile_trapping_restore(void) {feenableexcept(TRAPS);long long last;if (g_terminal_buddy.is_terminal &&- (last = atomic_load_explicit(&g_underflowed, memory_order_relaxed))) {+ (last = g_underflowed.load(std::memory_order_relaxed))) {long long now = millis();if (now - last > UNDERFLOW_DELAY &&- now - atomic_exchange_explicit(&g_underflowed, 0, memory_order_relaxed) >- UNDERFLOW_DELAY) {+ now - g_underflowed.exchange(0, std::memory_order_relaxed) > UNDERFLOW_DELAY) {write(2, UNDERFLOW_RESET, strlen(UNDERFLOW_RESET));}}diff --git a/llamafile/flags.cpp b/llamafile/flags.cppindex 975c026d..c36f208a 100644--- a/llamafile/flags.cpp+++ b/llamafile/flags.cpp@@ -15,6 +15,283 @@// See the License for the specific language governing permissions and// limitations under the License.+#include "debug.h"#include "llamafile.h"-bool FLAG_precise;+#include <errno.h>+#include <fcntl.h>+#include <stdio.h>+#include <stdlib.h>+#include <unistd.h>++#include "llama.cpp/llama.h"++bool FLAG_log_disable = false;+bool FLAG_mlock = false;+bool FLAG_mmap = true;+bool FLAG_nocompile = false;+bool FLAG_precise = false;+bool FLAG_recompile = false;+bool FLAG_tinyblas = false;+bool FLAG_unsecure = false;+const char *FLAG_file = nullptr;+const char *FLAG_listen = "0.0.0.0:8080";+const char *FLAG_model = nullptr;+const char *FLAG_prompt = nullptr;+float FLAG_temp = 0.8;+int FLAG_batch = 2048;+int FLAG_ctx = 512;+int FLAG_flash_attn = false;+int FLAG_gpu = 0;+int FLAG_keepalive = 5;+int FLAG_main_gpu = 0;+int FLAG_n_gpu_layers = -1;+int FLAG_seed = LLAMA_DEFAULT_SEED;+int FLAG_split_mode = LLAMA_SPLIT_MODE_LAYER;+int FLAG_threads = 64;+int FLAG_ubatch = 512;+int FLAG_verbose = 0;+int FLAG_workers = 0;++int cpu_get_num_math();++static wontreturn void usage(int rc, int fd) {+ tinyprint(fd, "usage: ", program_invocation_name, " -m MODEL -l [HOST:]PORT\n", NULL);+ exit(rc);+}++static wontreturn void error(const char *message) {+ tinyprint(2, program_invocation_name, ": ", message, "\n", NULL);+ exit(1);+}++static wontreturn void bad(const char *flag) {+ tinyprint(2, program_invocation_name, ": bad value for ", flag, "\n", NULL);+ exit(1);+}++static wontreturn void missing(const char *flag) {+ tinyprint(2, program_invocation_name, ": ", flag, " missing argument\n", NULL);+ exit(1);+}++static wontreturn void required(const char *flag) {+ tinyprint(2, program_invocation_name, ": ", flag, " is required\n", NULL);+ exit(1);+}++static wontreturn void unknown(const char *flag) {+ tinyprint(2, program_invocation_name, ": ", flag, " unknown argument\n", NULL);+ exit(1);+}++void llamafile_get_flags(int argc, char **argv) {+ FLAG_threads = cpu_get_num_math();+ for (int i = 1; i < argc;) {+ const char *flag = argv[i++];++ if (*flag != '-')+ break;++ //////////////////////////////////////////////////////////////////////+ // logging flags++ if (!strcmp(flag, "--log-disable")) {+ FLAG_log_disable = true;+ continue;+ }++ if (!strcmp(flag, "-v") || !strcmp(flag, "--verbose")) {+ FLAG_verbose++;+ continue;+ }++ //////////////////////////////////////////////////////////////////////+ // server flags++ if (!strcmp(flag, "-l") || !strcmp(flag, "--listen")) {+ if (i == argc)+ missing("--listen");+ FLAG_listen = argv[i++];+ continue;+ }++ if (!strcmp(flag, "-k") || !strcmp(flag, "--keepalive")) {+ if (i == argc)+ missing("--keepalive");+ FLAG_keepalive = atoi(argv[i++]);+ continue;+ }++ if (!strcmp(flag, "-w") || !strcmp(flag, "--workers")) {+ if (i == argc)+ missing("--workers");+ FLAG_workers = atoi(argv[i++]);+ continue;+ }++ //////////////////////////////////////////////////////////////////////+ // model flags++ if (!strcmp(flag, "-m") || !strcmp(flag, "--model")) {+ if (i == argc)+ missing("--model");+ FLAG_model = argv[i++];+ continue;+ }++ if (!strcmp(flag, "-f") || !strcmp(flag, "--file")) {+ if (i == argc)+ missing("--file");+ FLAG_file = argv[i++];+ continue;+ }++ if (!strcmp(flag, "--seed")) {+ if (i == argc)+ missing("--seed");+ FLAG_seed = atoi(argv[i++]);+ continue;+ }++ if (!strcmp(flag, "--temp")) {+ if (i == argc)+ missing("--temp");+ FLAG_temp = atof(argv[i++]);+ continue;+ }++ if (!strcmp(flag, "-t") || !strcmp(flag, "--threads")) {+ if (i == argc)+ missing("--threads");+ FLAG_threads = atoi(argv[i++]);+ continue;+ }++ if (!strcmp(flag, "-b") || !strcmp(flag, "--batch-size")) {+ if (i == argc)+ missing("--batch-size");+ FLAG_batch = atoi(argv[i++]);+ continue;+ }++ if (!strcmp(flag, "-ub") || !strcmp(flag, "--ubatch-size")) {+ if (i == argc)+ missing("--ubatch-size");+ FLAG_ubatch = atoi(argv[i++]);+ continue;+ }++ if (!strcmp(flag, "-fa") || !strcmp(flag, "--flash-attn")) {+ if (i == argc)+ missing("--flash-attn");+ FLAG_flash_attn = true;+ continue;+ }++ //////////////////////////////////////////////////////////////////////+ // cpu flags++ if (!strcmp(flag, "--fast")) {+ FLAG_precise = false;+ continue;+ }++ if (!strcmp(flag, "--precise")) {+ FLAG_precise = true;+ continue;+ }++ if (!strcmp(flag, "--trap")) {+ FLAG_trap = true;+ FLAG_unsecure = true;+ llamafile_trapping_enabled(+1);+ continue;+ }++ if (!strcmp(flag, "--mlock")) {+ FLAG_mlock = true;+ continue;+ }++ if (!strcmp(flag, "--no-mmap")) {+ FLAG_mmap = false;+ continue;+ }++ //////////////////////////////////////////////////////////////////////+ // gpu flags++ if (!strcmp(flag, "--tinyblas")) {+ FLAG_tinyblas = true;+ continue;+ }++ if (!strcmp(flag, "--nocompile")) {+ FLAG_nocompile = true;+ continue;+ }++ if (!strcmp(flag, "--recompile")) {+ FLAG_recompile = true;+ continue;+ }++ if (!strcmp(flag, "--gpu")) {+ if (i == argc)+ missing("--gpu");+ FLAG_gpu = llamafile_gpu_parse(argv[i++]);+ if (FLAG_gpu == LLAMAFILE_GPU_ERROR)+ bad("--gpu");+ continue;+ }++ if (!strcmp(flag, "-ngl") || //+ !strcmp(flag, "--gpu-layers") || //+ !strcmp(flag, "--n-gpu-layers")) {+ if (i == argc)+ missing("--n-gpu-layers");+ FLAG_n_gpu_layers = atoi(argv[i++]);+ if (FLAG_n_gpu_layers <= 0)+ FLAG_gpu = LLAMAFILE_GPU_DISABLE;+ continue;+ }++ if (!strcmp(flag, "-mg") || !strcmp(flag, "--main-gpu")) {+ if (i == argc)+ missing("--main-gpu");+ FLAG_main_gpu = atoi(argv[i++]);+ continue;+ }++ if (!strcmp(flag, "-sm") || !strcmp(flag, "--split-mode")) {+ if (i == argc)+ missing("--split-mode");+ const char *value = argv[i];+ if (!strcmp(value, "none"))+ FLAG_split_mode = LLAMA_SPLIT_MODE_NONE;+ else if (!strcmp(value, "layer"))+ FLAG_split_mode = LLAMA_SPLIT_MODE_LAYER;+ else if (!strcmp(value, "row"))+ FLAG_split_mode = LLAMA_SPLIT_MODE_ROW;+ else+ bad("--split-mode");+ continue;+ }++ //////////////////////////////////////////////////////////////////////+ // security flags++ if (!strcmp(flag, "--unsecure")) {+ FLAG_unsecure = true;+ continue;+ }++ unknown(flag);+ }++ if (!FLAG_model)+ required("--model");++ FLAG_n_gpu_layers = llamafile_gpu_layers(FLAG_n_gpu_layers);+}diff --git a/llamafile/gpu.c b/llamafile/gpu.cindex dd2c7d67..757805f8 100644--- a/llamafile/gpu.c+++ b/llamafile/gpu.c@@ -24,12 +24,6 @@#include <string.h>#include <unistd.h>-int FLAG_gpu;-bool FLAG_nogpu;-bool FLAG_tinyblas;-bool FLAG_nocompile;-bool FLAG_recompile;-const char *llamafile_describe_gpu(void) {switch (FLAG_gpu) {case LLAMAFILE_GPU_AUTO:diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 3b2b7047..5b261b20 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -6,6 +6,35 @@extern "C" {#endif+extern bool FLAG_log_disable;+extern bool FLAG_mlock;+extern bool FLAG_mmap;+extern bool FLAG_nocompile;+extern bool FLAG_precise;+extern bool FLAG_recompile;+extern bool FLAG_tinyblas;+extern bool FLAG_trap;+extern bool FLAG_unsecure;+extern const char *FLAG_file;+extern const char *FLAG_listen;+extern const char *FLAG_model;+extern const char *FLAG_prompt;+extern float FLAG_temp;+extern int FLAG_batch;+extern int FLAG_ctx;+extern int FLAG_flash_attn;+extern int FLAG_gpu;+extern int FLAG_gpu;+extern int FLAG_keepalive;+extern int FLAG_main_gpu;+extern int FLAG_n_gpu_layers;+extern int FLAG_seed;+extern int FLAG_split_mode;+extern int FLAG_threads;+extern int FLAG_ubatch;+extern int FLAG_verbose;+extern int FLAG_workers;+struct llamafile;struct llamafile *llamafile_open_gguf(const char *, const char *);void llamafile_close(struct llamafile *);@@ -18,6 +47,7 @@ size_t llamafile_size(struct llamafile *);FILE *llamafile_fp(struct llamafile *);void llamafile_ref(struct llamafile *);void llamafile_unref(struct llamafile *);+char *llamafile_get_prompt(void);void llamafile_govern(void);void llamafile_check_cpu(void);@@ -30,10 +60,7 @@ int llamafile_is_file_newer_than(const char *, const char *);void llamafile_schlep(const void *, size_t);void llamafile_get_app_dir(char *, size_t);void llamafile_launch_browser(const char *);--extern bool FLAG_trap;-extern bool FLAG_precise;-extern bool FLAG_unsecure;+void llamafile_get_flags(int, char **);#define LLAMAFILE_GPU_ERROR -2#define LLAMAFILE_GPU_DISABLE -1@@ -41,10 +68,6 @@ extern bool FLAG_unsecure;#define LLAMAFILE_GPU_AMD 1#define LLAMAFILE_GPU_APPLE 2#define LLAMAFILE_GPU_NVIDIA 4-extern int FLAG_gpu;-extern bool FLAG_tinyblas;-extern bool FLAG_nocompile;-extern bool FLAG_recompile;bool llamafile_has_gpu(void);int llamafile_gpu_layers(int);bool llamafile_has_cuda(void);diff --git a/llamafile/log.c b/llamafile/log.cindex b32f0431..47301eb5 100644--- a/llamafile/log.c+++ b/llamafile/log.c@@ -16,12 +16,12 @@// limitations under the License.#include "log.h"+#include "llamafile.h"+#include <pthread.h>#include <string.h>#include <unistd.h>-bool FLAG_log_disable;-void(tinylog)(const char *s, ...) {size_t n;int c, cs;diff --git a/llamafile/server/.clang-format b/llamafile/server/.clang-formatnew file mode 100644index 00000000..5c2c5d23--- /dev/null+++ b/llamafile/server/.clang-format@@ -0,0 +1,13 @@+---+BasedOnStyle: Mozilla+IndentWidth: 4+ColumnLimit: 80+---+Language: Cpp+AllowShortFunctionsOnASingleLine: false+AlignTrailingComments: false+AlignEscapedNewlines: DontAlign+AlwaysBreakTemplateDeclarations: true+ConstructorInitializerAllOnOneLineOrOnePerLine: true+FixNamespaceComments: true+---diff --git a/llamafile/server/BUILD.mk b/llamafile/server/BUILD.mknew file mode 100644index 00000000..10cab956--- /dev/null+++ b/llamafile/server/BUILD.mk@@ -0,0 +1,34 @@+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐+#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘++PKGS += LLAMAFILE_SERVER++LLAMAFILE_SERVER_FILES := $(wildcard llamafile/server/*)+LLAMAFILE_SERVER_HDRS = $(filter %.h,$(LLAMAFILE_SERVER_FILES))+LLAMAFILE_SERVER_SRCS = $(filter %.cpp,$(LLAMAFILE_SERVER_FILES))+LLAMAFILE_SERVER_OBJS = $(LLAMAFILE_SERVER_SRCS:%.cpp=o/$(MODE)/%.o)++o/$(MODE)/llamafile/server/server.a: \+ $(filter-out %_test.o,$(LLAMAFILE_SERVER_OBJS))++o/$(MODE)/llamafile/server/main: \+ o/$(MODE)/llamafile/server/main.o \+ o/$(MODE)/llamafile/server/server.a \+ o/$(MODE)/llama.cpp/llama.cpp.a \+ o/$(MODE)/llama.cpp/llava/llava.a \+ o/$(MODE)/double-conversion/double-conversion.a \+ o/$(MODE)/stb/stb.a \+ o/$(MODE)/llamafile/server/malloc.o \++$(LLAMAFILE_SERVER_OBJS): llamafile/server/BUILD.mk+$(LLAMAFILE_SERVER_OBJS): private CCFLAGS += -O++o/$(MODE)/llamafile/server/json_test: \+ o/$(MODE)/llamafile/server/json_test.o \+ o/$(MODE)/llamafile/server/json.o \+ o/$(MODE)/double-conversion/double-conversion.a \++.PHONY: o/$(MODE)/llamafile/server+o/$(MODE)/llamafile/server: \+ o/$(MODE)/llamafile/server/main \+ o/$(MODE)/llamafile/server/json_test.runs \diff --git a/llamafile/server/atob.cpp b/llamafile/server/atob.cppnew file mode 100644index 00000000..ee0dd3ee--- /dev/null+++ b/llamafile/server/atob.cpp@@ -0,0 +1,32 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "utils.h"++bool+atob(ctl::string_view val, bool dflt)+{+ if (val == "0")+ return false;+ if (val == "false")+ return false;+ if (val == "1")+ return true;+ if (val == "true")+ return true;+ return dflt;+}diff --git a/llamafile/server/buffer.cpp b/llamafile/server/buffer.cppnew file mode 100644index 00000000..65bfc349--- /dev/null+++ b/llamafile/server/buffer.cpp@@ -0,0 +1,48 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "buffer.h"++#include <sys/auxv.h>+#include <sys/mman.h>++static int pagesz = getauxval(AT_PAGESZ);++Buffer::Buffer(size_t capacity) noexcept+ : i(0)+ , n(0)+ , c(capacity - pagesz)+ , p((char*)mmap(nullptr,+ capacity,+ PROT_READ | PROT_WRITE,+ MAP_PRIVATE | MAP_ANONYMOUS,+ -1,+ 0))+{+ if (p == MAP_FAILED)+ __builtin_trap();+ if (c & (pagesz - 1))+ __builtin_trap();+ if (mprotect(p + c, pagesz, PROT_NONE))+ __builtin_trap();+}++Buffer::~Buffer() noexcept+{+ if (munmap(p, c + pagesz))+ __builtin_trap();+}diff --git a/llamafile/server/buffer.h b/llamafile/server/buffer.hnew file mode 100644index 00000000..757a5d15--- /dev/null+++ b/llamafile/server/buffer.h@@ -0,0 +1,30 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <stddef.h>++struct Buffer+{+ size_t i;+ size_t n;+ const size_t c;+ char* const p;++ explicit Buffer(size_t) noexcept;+ ~Buffer() noexcept;+};diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cppnew file mode 100644index 00000000..b08b9f96--- /dev/null+++ b/llamafile/server/client.cpp@@ -0,0 +1,494 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "client.h"++#include <ctl/optional.h>+#include <ctl/string.h>+#include <errno.h>+#include <libc/fmt/itoa.h>+#include <libc/str/slice.h>+#include <limits.h>+#include <string.h>+#include <sys/auxv.h>+#include <sys/uio.h>+#include <time.h>+#include <unistd.h>++#include "json.h"+#include "llama.cpp/llama.h"+#include "llamafile/llamafile.h"+#include "llamafile/version.h"+#include "log.h"+#include "time.h"+#include "utils.h"++#define STANDARD_RESPONSE_HEADERS \+ "Server: llamafile/" LLAMAFILE_VERSION_STRING "\r\n" \+ "Referrer-Policy: origin\r\n" \+ "Cache-Control: private; max-age=0\r\n"++#define HasHeader(H) (!!msg.headers[H].a)+#define HeaderData(H) (ibuf.p + msg.headers[H].a)+#define HeaderLength(H) (msg.headers[H].b - msg.headers[H].a)+#define HeaderEqual(H, S) \+ SlicesEqual(S, strlen(S), HeaderData(H), HeaderLength(H))+#define HeaderEqualCase(H, S) \+ SlicesEqualCase(S, strlen(S), HeaderData(H), HeaderLength(H))++using namespace ctl;++Client::Client() : ibuf(8 * 1024 * 1024), obuf(64 * 1024 * 1024)+{+ InitHttpMessage(&msg, 0);+ url.params.p = nullptr;+}++int+Client::close()+{+ int rc = 0;+ clear();+ DestroyHttpMessage(&msg);+ if (fd != -1) {+ if (FLAG_verbose >= 2)+ LOG("close");+ rc = ::close(fd);+ fd = -1;+ }+ return rc;+}++void+Client::clear()+{+ free(url_memory);+ url_memory = nullptr;+ free(params_memory);+ params_memory = nullptr;+ free(url.params.p);+ url.params.p = nullptr;+ close_connection = false;+ payload = "";+ unread = 0;+}++void+Client::run()+{+ ibuf.n = 0;+ for (;;) {++ // read headers+ clear();+ if (!read_request())+ break;++ // process message+ if (!transport())+ break;++ // synchronize message stream+ if (close_connection)+ break;+ if (!read_payload())+ break;++ // move pipelined bytes back to beginning+ if (ibuf.n == ibuf.i) {+ ibuf.n = 0;+ } else {+ memmove(ibuf.p, ibuf.p + ibuf.i, ibuf.n - ibuf.i);+ ibuf.n -= ibuf.i;+ }+ }+}++bool+Client::read_request()+{+ int inmsglen;+ ResetHttpMessage(&msg, kHttpRequest);+ for (;;) {+ inmsglen = ParseHttpMessage(&msg, ibuf.p, ibuf.n, ibuf.c);+ if (inmsglen > 0) {+ message_started = timespec_real();+ ibuf.i = inmsglen;+ return true;+ }+ if (inmsglen == -1) {+ LOG("bad message %m");+ return false;+ }+ if (ibuf.n)+ LOG("fragmented message with %zu bytes", ibuf.n);+ ssize_t got;+ got = read(fd, ibuf.p + ibuf.n, ibuf.c - ibuf.n);+ if (!got && ibuf.n)+ LOG("unexpected eof after %zu bytes", ibuf.n);+ if (got == -1 && (ibuf.n || (errno != EAGAIN && errno != ECONNRESET)))+ LOG("read failed %m");+ if (got <= 0)+ return false;+ ibuf.n += got;+ }+}++bool+Client::transport()+{+ if (msg.version > 11) {+ close_connection = true;+ return send_error(505);+ }++ if (msg.method == kHttpConnect) {+ close_connection = true;+ return send_error(501);+ }++ if (!has_at_most_this_element(kHttpExpect, "100-continue")) {+ close_connection = true;+ return send_error(417);+ }++ if (HasHeader(kHttpTransferEncoding))+ if (!HeaderEqualCase(kHttpTransferEncoding, "identity")) {+ close_connection = true;+ return send_error(501, "Transfer-Encoding Not Implemented");+ }++ if (HasHeader(kHttpContentLength)) {+ long cl;+ cl = ParseContentLength(HeaderData(kHttpContentLength),+ HeaderLength(kHttpContentLength));+ if (cl == -1) {+ close_connection = true;+ return send_error(400, "Bad Content-Length");+ }+ if (cl > ibuf.c - ibuf.i) {+ close_connection = true;+ return send_error(413);+ }+ unread = cl;+ } else if (msg.method == kHttpPost || msg.method == kHttpPut) {+ close_connection = true;+ return send_error(411);+ }++ if (FLAG_verbose >= 1)+ LOG("get %#.*s", msg.uri.b - msg.uri.a, ibuf.p + msg.uri.a);++ if (msg.version >= 11)+ if (HeaderEqualCase(kHttpExpect, "100-continue"))+ if (!send("HTTP/1.1 100 Continue\r\n\r\n"))+ return false;++ url_memory = ParseUrl(+ ibuf.p + msg.uri.a, msg.uri.b - msg.uri.a, &url, kUrlPlus | kUrlLatin1);+ if (!url_memory)+ __builtin_trap();++ return dispatch();+}++bool+Client::send_error(int code, const char* reason)+{+ if (!reason)+ reason = GetHttpReason(code);+ LOG("error %d %s", code, reason);+ char* p = start_response(obuf.p, code, reason);+ return send_response(obuf.p, p, string(reason) + "\r\n");+}++char*+Client::start_response(char* p, int code, const char* reason)+{+ *p++ = 'H';+ *p++ = 'T';+ *p++ = 'T';+ *p++ = 'P';+ *p++ = '/';+ *p++ = '1';+ *p++ = '.';+ *p++ = '0' + (msg.version & 1);+ *p++ = ' ';+ *p++ = '0' + code / 100;+ *p++ = '0' + code / 10 % 10;+ *p++ = '0' + code % 10;+ *p++ = ' ';+ if (!reason)+ reason = GetHttpReason(code);+ p = stpcpy(p, reason);+ *p++ = '\r';+ *p++ = '\n';+ p = stpcpy(p, STANDARD_RESPONSE_HEADERS);+ return p;+}++bool+Client::send_response(char* p0, char* p, string_view content)+{+ // append date header+ tm tm;+ p = stpcpy(p, "Date: ");+ gmtime_lockless(message_started.tv_sec, &tm);+ p = FormatHttpDateTime(p, &tm);+ *p++ = '\r';+ *p++ = '\n';++ // inform client of close() intent+ if (msg.version < 11)+ close_connection = true;+ if (HeaderEqualCase(kHttpConnection, "close"))+ close_connection = true;+ if (close_connection)+ p = stpcpy(p, "Connection: close\r\n");++ // append content length+ p = stpcpy(p, "Content-Length: ");+ p = FormatInt64(p, content.size());+ *p++ = '\r';+ *p++ = '\n';++ // finish message+ *p++ = '\r';+ *p++ = '\n';++ return send2(string_view(p0, p - p0), content);+}++bool+Client::send(const string_view s)+{+ ssize_t sent;+ if ((sent = write(fd, s.data(), s.size())) != s.size()) {+ if (sent == -1 && errno != EAGAIN && errno != ECONNRESET)+ LOG("write failed %m");+ return false;+ }+ return true;+}++bool+Client::send2(const string_view s1, const string_view s2)+{+ iovec iov[2];+ ssize_t sent;+ iov[0].iov_base = (void*)s1.data();+ iov[0].iov_len = s1.size();+ iov[1].iov_base = (void*)s2.data();+ iov[1].iov_len = s2.size();+ if ((sent = writev(fd, iov, 2)) != s1.size() + s2.size()) {+ if (sent == -1 && errno != EAGAIN && errno != ECONNRESET)+ LOG("writev failed %m");+ return false;+ }+ return true;+}++bool+Client::has_at_most_this_element(int h, const string_view s)+{+ if (!HasHeader(h))+ return true;+ if (!SlicesEqualCase(s.data(), s.size(), HeaderData(h), HeaderLength(h)))+ return false;+ struct HttpHeader* x;+ for (unsigned i = 0; i < msg.xheaders.n; ++i) {+ x = msg.xheaders.p + i;+ if (GetHttpHeader(ibuf.p + x->k.a, x->k.b - x->k.a) == h &&+ !SlicesEqualCase(+ ibuf.p + x->v.a, x->v.b - x->v.a, s.data(), s.size())) {+ return false;+ }+ }+ return true;+}++bool+Client::read_payload()+{+ while (ibuf.n - ibuf.i < unread) {+ ssize_t got;+ if ((got = read(fd, ibuf.p + ibuf.n, ibuf.c - ibuf.n)) <= 0) {+ if (!got)+ LOG("unexpected eof");+ if (got == -1)+ LOG("read failed %m");+ return false;+ }+ ibuf.n += got;+ }+ payload = string_view(ibuf.p + ibuf.i, unread);+ ibuf.i += unread;+ unread = 0;+ if (msg.method == kHttpPost && //+ HasHeader(kHttpContentType) &&+ IsMimeType(HeaderData(kHttpContentType),+ HeaderLength(kHttpContentType),+ "application/x-www-form-urlencoded")) {+ params_memory =+ ParseParams(payload.data(), payload.size(), &url.params);+ }++ return true;+}++bool+Client::dispatch()+{+ if (path() == "/tokenize")+ return tokenize();+ return send_error(404);+}++string_view+Client::path()+{+ if (!url.path.n)+ return "/";+ return { url.path.p, url.path.n };+}++optional<string_view>+Client::param(string_view key)+{+ for (size_t i = 0; i < url.params.n; ++i)+ if (key.size() == url.params.p[i].key.n)+ if (!memcmp(key.data(), url.params.p[i].key.p, key.size()))+ return optional(+ string_view(url.params.p[i].val.p, url.params.p[i].val.n));+ return {};+}++static string_view+or_empty(optional<string_view> x)+{+ if (x.has_value())+ return x.value();+ return {};+}++bool+Client::tokenize()+{+ if (msg.method != kHttpGet && msg.method != kHttpPost)+ return send_error(405);++ if (!read_payload())+ return false;++ // get prompt+ //+ // 1. Allow GET "/tokenize?prompt=foo"+ // 2. Allow POST "prompt=foo" (application/x-www-form-urlencoded)+ // 3. Allow POST "foo" (text/plain)+ //+ string_view input;+ optional<string_view> prompt = param("prompt");+ if (prompt.has_value()) {+ input = prompt.value();+ } else if (HasHeader(kHttpContentType)) {+ if (IsMimeType(HeaderData(kHttpContentType),+ HeaderLength(kHttpContentType),+ "text/plain")) {+ input = payload;+ } else {+ return send_error(501, "Content Type Not Implemented");+ }+ } else {+ input = payload;+ }++ // get optional parameters+ bool add_special = atob(or_empty(param("add_special")), true);+ bool parse_special = atob(or_empty(param("parse_special")), false);++ // setup statistics+ rusage rustart = {};+ getrusage(RUSAGE_THREAD, &rustart);+ timespec started = timespec_real();++ // turn text into tokens+ extern llama_model* g_model;+ int maxcount = input.size() + 16;+ llama_token* toks = new llama_token[maxcount];+ int count = llama_tokenize(g_model,+ input.data(),+ input.size(),+ toks,+ maxcount,+ add_special,+ parse_special);+ if (count < 0) {+ delete[] toks;+ __builtin_trap();+ }++ // serialize tokens to json+ char* p = obuf.p;+ p = stpcpy(p, "{\r\n");+ p = stpcpy(p, " \"add_special\": ");+ p = encode_bool(p, add_special);+ p = stpcpy(p, ",\n");+ p = stpcpy(p, " \"parse_special\": ");+ p = encode_bool(p, parse_special);+ p = stpcpy(p, ",\n");+ p = stpcpy(p, " \"tokens\": [");+ for (int i = 0; i < count; ++i) {+ if (i)+ *p++ = ',';+ p = stpcpy(p, "\r\n ");+ char s[32];+ int n = llama_token_to_piece(g_model, toks[i], s, sizeof(s), true);+ if (n < 0) {+ delete[] toks;+ __builtin_trap();+ }+ p = encode_json(p, string_view(s, n));+ }+ p = stpcpy(p, "\r\n ]\r\n");+ p = stpcpy(p, "}\r\n");+ string_view content(obuf.p, p - obuf.p);+ delete[] toks;++ // collect statistics+ rusage ruend = {};+ getrusage(RUSAGE_THREAD, &ruend);+ timeval user = timeval_sub(ruend.ru_utime, rustart.ru_utime);+ timeval system = timeval_sub(ruend.ru_stime, rustart.ru_stime);+ timespec ended = timespec_real();+ timespec wall = timespec_sub(ended, started);+ long wall_us = timespec_tomicros(wall);+ long user_us = timeval_tomicros(user);+ long system_us = timeval_tomicros(system);++ // send response+ char* headers = p;+ p = start_response(p, 200);+ p = stpcpy(p, "Content-Type: application/json\r\n");+ p = stpcpy(p, "X-Wall-Micros: ");+ p = FormatInt64(p, wall_us);+ p = stpcpy(p, "\r\nX-User-Micros: ");+ p = FormatInt64(p, user_us);+ p = stpcpy(p, "\r\nX-System-Micros: ");+ p = FormatInt64(p, system_us);+ p = stpcpy(p, "\r\n");+ return send_response(headers, p, content);+}diff --git a/llamafile/server/client.h b/llamafile/server/client.hnew file mode 100644index 00000000..2234c874--- /dev/null+++ b/llamafile/server/client.h@@ -0,0 +1,67 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once++#include "buffer.h"++#include <ctl/optional.h>+#include <ctl/string_view.h>+#include <net/http/http.h>+#include <net/http/url.h>+#include <sys/resource.h>+#include <time.h>++struct Client+{+ int fd = -1;+ bool close_connection = false;+ size_t unread = 0;+ timespec message_started;+ HttpMessage msg;+ Url url = {};+ char* url_memory = nullptr;+ char* params_memory = nullptr;+ ctl::string_view payload;+ Buffer ibuf;+ Buffer obuf;++ Client();++ void run();+ int close();+ void clear();+ bool transport() __wur;+ bool synchronize() __wur;+ bool read_payload() __wur;+ bool read_request() __wur;+ bool read_content() __wur;+ bool send_continue() __wur;+ bool send(const ctl::string_view) __wur;+ char* start_response(char*, int, const char* = nullptr);+ bool send_error(int, const char* = nullptr) __wur;+ bool send_response(char*, char*, const ctl::string_view) __wur;+ bool send2(const ctl::string_view, const ctl::string_view) __wur;+ char* append_header(const ctl::string_view, const ctl::string_view);+ bool has_at_most_this_element(int, const ctl::string_view);++ ctl::string_view path();+ ctl::optional<ctl::string_view> param(ctl::string_view);++ bool dispatch() __wur;+ bool tokenize() __wur;+};diff --git a/llamafile/server/crash.cpp b/llamafile/server/crash.cppnew file mode 100644index 00000000..3aa9f2c8--- /dev/null+++ b/llamafile/server/crash.cpp@@ -0,0 +1,128 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "signals.h"+#include "utils.h"++#include <assert.h>+#include <cosmo.h>+#include <ucontext.h>++#ifdef __aarch64__+#define PC pc+#define BP regs[29]+#else+#define PC gregs[REG_RIP]+#define BP gregs[REG_RBP]+#endif++// returns true if `p` is preceded by x86 call instruction+// this is actually impossible to do but we'll do our best+int+is_call(const unsigned char* p)+{+ if (p[-5] == 0xe8)+ return 5; // call Jvds+ if (p[-2] == 0xff && (p[-1] & 070) == 020)+ return 2; // call %reg+ if (p[-4] == 0xff && (p[-3] & 070) == 020)+ return 4; // call disp8(%reg,%reg)+ if (p[-3] == 0xff && (p[-2] & 070) == 020)+ return 3; // call disp8(%reg)+ if (p[-7] == 0xff && (p[-6] & 070) == 020)+ return 7; // call disp32(%reg,%reg)+ if (p[-6] == 0xff && (p[-5] & 070) == 020)+ return 6; // call disp32(%reg)+ return 0;+}++void+describe_crash(char* buf, size_t len, int sig, siginfo_t* si, void* arg)+{+ unassert(len >= 64);++ // describe crash+ char* p = buf;+ char* pe = p + len;+ char signame[21];+ p = stpcpy(p, strsignal_r(sig, signame));+ if (si && //+ (sig == SIGFPE || //+ sig == SIGILL || //+ sig == SIGBUS || //+ sig == SIGSEGV || //+ sig == SIGTRAP)) {+ p = stpcpy(p, " at ");+ p = hexcpy(p, (long)si->si_addr);+ }++ // get stack frame daisy chain+ long pc;+ ucontext_t* ctx;+ struct StackFrame* sf;+ if ((ctx = (ucontext_t*)arg)) {+ pc = ctx->uc_mcontext.PC;+ sf = (struct StackFrame*)ctx->uc_mcontext.BP;+ } else {+ pc = 0;+ sf = (struct StackFrame*)__builtin_frame_address(0);+ }++ // describe backtrace+ p = stpcpy(p, " bt ");+ if (pc) {+ p = hexcpy(p, pc);+ *p++ = ' ';+ }+ bool gotsome = false;+ while (sf) {+ if (kisdangerous(sf)) {+ if (p + 1 + 9 + 1 < pe) {+ if (gotsome)+ *p++ = ' ';+ p = stpcpy(p, "DANGEROUS");+ if (p + 16 + 1 < pe) {+ *p++ = ' ';+ p = hexcpy(p, (long)sf);+ }+ }+ break;+ }+ if (p + 16 + 1 < pe) {+ unsigned char* ip = (unsigned char*)sf->addr;+#ifdef __x86_64__+ // x86 advances the progrem counter before an instruction+ // begins executing. return addresses in backtraces shall+ // point to code after the call, which means addr2line is+ // going to print unrelated code unless we fixup the addr+ if (!kisdangerous(ip))+ ip -= is_call(ip);+#endif+ if (gotsome)+ *p++ = ' ';+ else+ gotsome = true;+ p = hexcpy(p, (long)ip);+ } else {+ break;+ }+ sf = sf->next;+ }++ // terminate string+ *p = '\0';+}diff --git a/llamafile/server/hexcpy.cpp b/llamafile/server/hexcpy.cppnew file mode 100644index 00000000..bc53d507--- /dev/null+++ b/llamafile/server/hexcpy.cpp@@ -0,0 +1,29 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "utils.h"++char*+hexcpy(char* p, unsigned long x)+{+ int k = x ? (__builtin_clzl(x) ^ 63) + 1 : 1;+ k = (k + 3) & -4;+ while (k > 0)+ *p++ = "0123456789abcdef"[(x >> (k -= 4)) & 15];+ *p = '\0';+ return p;+}diff --git a/llamafile/server/hextoint.cpp b/llamafile/server/hextoint.cppnew file mode 100644index 00000000..f6bec4f7--- /dev/null+++ b/llamafile/server/hextoint.cpp@@ -0,0 +1,37 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "utils.h"++alignas(signed char) const signed char kHexToInt[256] = {+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x00+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x10+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x20+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, // 0x30+ -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x40+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x50+ -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x60+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x70+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x80+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x90+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xa0+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xb0+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xc0+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xd0+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xe0+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xf0+};diff --git a/llamafile/server/json.cpp b/llamafile/server/json.cppnew file mode 100644index 00000000..8f03c57b--- /dev/null+++ b/llamafile/server/json.cpp@@ -0,0 +1,172 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "json.h"++#include <cosmo.h>+#include <ctl/string.h>+#include <net/http/escape.h>++#include "double-conversion/double-to-string.h"+#include "double-conversion/utils.h"++static const char kEscapeLiteral[128] = {+ 9, 9, 9, 9, 9, 9, 9, 9, 9, 1, 2, 9, 4, 3, 9, 9, // 0x00+ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 0x10+ 0, 0, 7, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 6, // 0x20+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, // 0x30+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x40+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, // 0x50+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x60+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, // 0x70+};++static const double_conversion::DoubleToStringConverter kDoubleToJson(+ double_conversion::DoubleToStringConverter::UNIQUE_ZERO |+ double_conversion::DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN,+ "1e5000",+ "null",+ 'e',+ -6,+ 21,+ 6,+ 0);++char*+encode_bool(char* p, bool x) noexcept+{+ return stpcpy(p, x ? "true" : "false");+}++char*+encode_json(char* p, int x) noexcept+{+ return FormatInt32(p, x);+}++char*+encode_json(char* p, long x) noexcept+{+ return FormatInt64(p, x);+}++char*+encode_json(char* p, unsigned x) noexcept+{+ return FormatUint32(p, x);+}++char*+encode_json(char* p, unsigned long x) noexcept+{+ return FormatUint64(p, x);+}++char*+encode_json(char* p, double x) noexcept+{+ double_conversion::StringBuilder b(p, 128);+ kDoubleToJson.ToShortest(x, &b);+ b.Finalize();+ return p + strlen(p);+}++char*+encode_json(char* p, const ctl::string_view s) noexcept+{+ *p++ = '"';+ p = encode_js_string_literal(p, s);+ *p++ = '"';+ *p = 0;+ return p;+}++char*+encode_js_string_literal(char* p, const ctl::string_view s) noexcept+{+ uint64_t w;+ size_t i, j, m;+ wint_t x, a, b;+ for (size_t i = 0; i < s.size();) {+ x = s[i++] & 255;+ if (x >= 0300) {+ a = ThomPikeByte(x);+ m = ThomPikeLen(x) - 1;+ if (i + m <= s.size()) {+ for (j = 0;;) {+ b = s[i + j] & 0xff;+ if (!ThomPikeCont(b))+ break;+ a = ThomPikeMerge(a, b);+ if (++j == m) {+ x = a;+ i += j;+ break;+ }+ }+ }+ }+ switch (0 <= x && x <= 127 ? kEscapeLiteral[x] : 9) {+ case 0:+ *p++ = x;+ break;+ case 1:+ *p++ = '\\';+ *p++ = 't';+ break;+ case 2:+ *p++ = '\\';+ *p++ = 'n';+ break;+ case 3:+ *p++ = '\\';+ *p++ = 'r';+ break;+ case 4:+ *p++ = '\\';+ *p++ = 'f';+ break;+ case 5:+ *p++ = '\\';+ *p++ = '\\';+ break;+ case 6:+ *p++ = '\\';+ *p++ = '/';+ break;+ case 7:+ *p++ = '\\';+ *p++ = '"';+ break;+ case 9:+ w = EncodeUtf16(x);+ do {+ *p++ = '\\';+ *p++ = 'u';+ *p++ = "0123456789abcdef"[(w & 0xF000) >> 014];+ *p++ = "0123456789abcdef"[(w & 0x0F00) >> 010];+ *p++ = "0123456789abcdef"[(w & 0x00F0) >> 004];+ *p++ = "0123456789abcdef"[(w & 0x000F) >> 000];+ } while ((w >>= 16));+ break;+ default:+ __builtin_unreachable();+ }+ }+ *p = 0;+ return p;+}diff --git a/llamafile/server/json.h b/llamafile/server/json.hnew file mode 100644index 00000000..99ab5e0d--- /dev/null+++ b/llamafile/server/json.h@@ -0,0 +1,43 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <ctl/string_view.h>++char*+encode_bool(char*, bool) noexcept;++char*+encode_json(char*, int) noexcept;++char*+encode_json(char*, long) noexcept;++char*+encode_json(char*, double) noexcept;++char*+encode_json(char*, unsigned) noexcept;++char*+encode_json(char*, unsigned long) noexcept;++char*+encode_json(char*, const ctl::string_view) noexcept;++char*+encode_js_string_literal(char*, const ctl::string_view) noexcept;diff --git a/llamafile/server/json_test.cpp b/llamafile/server/json_test.cppnew file mode 100644index 00000000..cf9d6647--- /dev/null+++ b/llamafile/server/json_test.cpp@@ -0,0 +1,128 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "json.h"++#include <ctl/string.h>+#include <limits.h>+#include <math.h>+#include <stdio.h>++using namespace ctl;++string+encode_json(int x) noexcept+{+ char buf[12];+ return { buf, (size_t)(encode_json(buf, x) - buf) };+}++string+encode_json(long x) noexcept+{+ char buf[21];+ return { buf, (size_t)(encode_json(buf, x) - buf) };+}++string+encode_json(double x) noexcept+{+ char buf[128];+ return { buf, (size_t)(encode_json(buf, x) - buf) };+}++string+encode_json(unsigned x) noexcept+{+ char buf[128];+ return { buf, (size_t)(encode_json(buf, x) - buf) };+}++string+encode_json(unsigned long x) noexcept+{+ char buf[128];+ return { buf, (size_t)(encode_json(buf, x) - buf) };+}++string+encode_json(const string_view x) noexcept+{+ char buf[128];+ return { buf, (size_t)(encode_json(buf, x) - buf) };+}++string+encode_js_string_literal(const string_view x) noexcept+{+ char buf[256]; // this isn't secure (no guard page)+ return { buf, (size_t)(encode_json(buf, x) - buf) };+}++int+main(int argc, char* argv[])+{++ if (encode_json(0) != "0")+ return 1;+ if (encode_json(INT_MAX) != "2147483647")+ return 2;+ if (encode_json(INT_MIN) != "-2147483648")+ return 3;+ if (encode_json(UINT_MAX) != "4294967295")+ return 4;+ if (encode_json(LONG_MAX) != "9223372036854775807")+ return 5;+ if (encode_json(LONG_MIN) != "-9223372036854775808")+ return 6;+ if (encode_json(ULONG_MAX) != "18446744073709551615")+ return 7;++ if (encode_json("") != "\"\"")+ return 8;+ if (encode_json(string_view("\0\1", 2)) != "\"\\u0000\\u0001\"")+ return 9;+ if (encode_json("\n\"\\\t") != "\"\\n\\\"\\\\\\t\"")+ return 10;+ if (encode_json("'") != "\"\\u0027\"")+ return 11;+ if (encode_json("µ") != "\"\\u00b5\"")+ return 12;+ if (encode_json("𐌰") != "\"\\ud800\\udf30\"")+ return 13;++ if (encode_json(3.) != "3")+ return 14;+ if (encode_json(3.14) != "3.14")+ return 15;+ if (encode_json(1e+100) != "1e+100")+ return 16;+ if (encode_json(1e-100) != "1e-100")+ return 17;+ if (encode_json(+INFINITY) != "1e5000")+ return 18;+ if (encode_json(-INFINITY) != "-1e5000")+ return 19;+ if (encode_json(+NAN) != "null")+ return 20;+ if (encode_json(-NAN) != "null")+ return 21;+ if (encode_json(1e-300) != "1e-300")+ return 21;++ return 0;+}diff --git a/llamafile/server/listen.cpp b/llamafile/server/listen.cppnew file mode 100644index 00000000..608589da--- /dev/null+++ b/llamafile/server/listen.cpp@@ -0,0 +1,114 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "server.h"++#include <netdb.h>+#include <netinet/in.h>+#include <netinet/tcp.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <sys/socket.h>++#include "log.h"++void+print_listening_url(unsigned ip, int port)+{+ LOG("listen http://%hhu.%hhu.%hhu.%hhu:%hu",+ ip >> 24,+ ip >> 16,+ ip >> 8,+ ip,+ port);+}++int+create_listening_socket(const char* hostport)+{+ // parse hostname:port+ char* p;+ char* host;+ char* port;+ char addr[128];+ strlcpy(addr, hostport, sizeof(addr));+ if ((p = strrchr(addr, ':'))) {+ *p = '\0';+ host = addr;+ port = p + 1;+ } else {+ host = NULL;+ port = addr;+ }++ // turn listen address names into numbers+ int status;+ struct addrinfo* ai;+ struct addrinfo hints = {+ .ai_family = AF_INET,+ .ai_socktype = SOCK_STREAM,+ .ai_protocol = IPPROTO_TCP,+ };+ if ((status = getaddrinfo(host, port, &hints, &ai))) {+ fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(status));+ exit(1);+ }++ // create socket+ int fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);+ if (fd == -1) {+ perror(hostport);+ exit(1);+ }++ // these fail on some platforms but it's harmless+ int yes = 1;+ int qlen = 5;+ setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));+ setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, &qlen, sizeof(qlen));++ // bind the socket+ if (bind(fd, (struct sockaddr*)ai->ai_addr, ai->ai_addrlen) == -1) {+ perror(hostport);+ exit(1);+ }++ // listen for connections+ if (listen(fd, SOMAXCONN)) {+ perror(hostport);+ exit(1);+ }++ // print listening urls+ if (getsockname(fd, (struct sockaddr*)ai->ai_addr, &ai->ai_addrlen)) {+ perror(hostport);+ exit(1);+ }+ struct sockaddr_in* in = (struct sockaddr_in*)ai->ai_addr;+ if (ntohl(in->sin_addr.s_addr) == INADDR_ANY) {+ int i;+ uint32_t* hostips;+ for (hostips = GetHostIps(), i = 0; hostips[i]; ++i)+ print_listening_url(hostips[i], ntohs(in->sin_port));+ } else {+ print_listening_url(ntohl(in->sin_addr.s_addr), ntohs(in->sin_port));+ }++ freeaddrinfo(ai);+ return fd;+}diff --git a/llamafile/server/log.cpp b/llamafile/server/log.cppnew file mode 100644index 00000000..b7834540--- /dev/null+++ b/llamafile/server/log.cpp@@ -0,0 +1,90 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "log.h"++#include <pthread.h>+#include <string.h>++#include "time.h"++static thread_local char g_thread_name[128];++const char*+get_thread_name(void)+{+ return g_thread_name;+}++void+set_thread_name(const char* name)+{+ char shortened[16];+ strlcpy(shortened, name, sizeof(shortened));+ pthread_setname_np(pthread_self(), shortened);+ strlcpy(g_thread_name, name, sizeof(g_thread_name));+}++char*+get_log_timestamp(void)+{+ tm tm;+ int x;+ timespec ts;+ thread_local static long last;+ thread_local static char s[27];+ clock_gettime(0, &ts);+ if (ts.tv_sec != last) {+ localtime_lockless(ts.tv_sec, &tm);+ x = tm.tm_year + 1900;+ s[0] = '0' + x / 1000;+ s[1] = '0' + x / 100 % 10;+ s[2] = '0' + x / 10 % 10;+ s[3] = '0' + x % 10;+ s[4] = '-';+ x = tm.tm_mon + 1;+ s[5] = '0' + x / 10;+ s[6] = '0' + x % 10;+ s[7] = '-';+ x = tm.tm_mday;+ s[8] = '0' + x / 10;+ s[9] = '0' + x % 10;+ s[10] = 'T';+ x = tm.tm_hour;+ s[11] = '0' + x / 10;+ s[12] = '0' + x % 10;+ s[13] = ':';+ x = tm.tm_min;+ s[14] = '0' + x / 10;+ s[15] = '0' + x % 10;+ s[16] = ':';+ x = tm.tm_sec;+ s[17] = '0' + x / 10;+ s[18] = '0' + x % 10;+ s[19] = '.';+ s[26] = 0;+ last = ts.tv_sec;+ }+ x = ts.tv_nsec;+ s[20] = '0' + x / 100000000;+ s[21] = '0' + x / 10000000 % 10;+ s[22] = '0' + x / 1000000 % 10;+ s[23] = '0' + x / 100000 % 10;+ s[24] = '0' + x / 10000 % 10;+ s[25] = '0' + x / 1000 % 10;+ return s;+}diff --git a/llamafile/server/log.h b/llamafile/server/log.hnew file mode 100644index 00000000..1ddb2bda--- /dev/null+++ b/llamafile/server/log.h@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <libc/intrin/kprintf.h>++#define LOG(FMT, ...) \+ kprintf("%s %s:%d %s " FMT "\n", \+ get_log_timestamp(), \+ __FILE__, \+ __LINE__, \+ get_thread_name(), \+ ##__VA_ARGS__)++const char*+get_thread_name(void);++char*+get_log_timestamp(void);++void+set_thread_name(const char*);diff --git a/llamafile/server/main.cpp b/llamafile/server/main.cppnew file mode 100644index 00000000..2e48aac1--- /dev/null+++ b/llamafile/server/main.cpp@@ -0,0 +1,90 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <assert.h>+#include <third_party/dlmalloc/dlmalloc.h>+#include <tool/args/args.h>++#include "llama.cpp/llama.h"+#include "llamafile/llamafile.h"+#include "llamafile/version.h"++#include "json.h"+#include "log.h"+#include "server.h"+#include "signals.h"+#include "time.h"++Server* g_server;+llama_model* g_model;++int+main(int argc, char* argv[])+{+ llamafile_check_cpu();+ if (llamafile_has(argv, "--version")) {+ puts("llamafile-server v" LLAMAFILE_VERSION_STRING);+ exit(0);+ }++ // get config+ LoadZipArgs(&argc, &argv);+ llamafile_get_flags(argc, argv);+ time_init();++ // load model+ llama_model_params mparams = {+ .n_gpu_layers = FLAG_n_gpu_layers,+ .split_mode = (enum llama_split_mode)FLAG_split_mode,+ .main_gpu = FLAG_main_gpu,+ .tensor_split = nullptr,+ .rpc_servers = nullptr,+ .progress_callback = nullptr,+ .progress_callback_user_data = nullptr,+ .kv_overrides = nullptr,+ .vocab_only = false,+ .use_mmap = true,+ .use_mlock = false,+ .check_tensors = false,+ };+ g_model = llama_load_model_from_file(FLAG_model, mparams);++ // create server+ if (FLAG_workers <= 0)+ FLAG_workers = __get_cpu_count();+ if (FLAG_workers <= 0)+ FLAG_workers = 16;+ set_thread_name("server");+ g_server = new Server(create_listening_socket(FLAG_listen));+ for (int i = 0; i < FLAG_workers; ++i)+ unassert(!g_server->spawn());++ // run server+ setup_signals();+ g_server->run();+ restore_signals();++ // shutdown server+ LOG("shutdown");+ g_server->shutdown();+ g_server->close();+ delete g_server;+ LOG("exit");++ // quality assurance+ CheckForMemoryLeaks();+}diff --git a/llamafile/server/path.cpp b/llamafile/server/path.cppnew file mode 100644index 00000000..c3aee5a8--- /dev/null+++ b/llamafile/server/path.cpp@@ -0,0 +1,67 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "path.h"++using namespace ctl;++string+basename(const string_view path) noexcept+{+ size_t i, e;+ if ((e = path.size())) {+ while (e > 1 && path[e - 1] == '/')+ --e;+ i = e - 1;+ while (i && path[i - 1] != '/')+ --i;+ return path.substr(i, e - i);+ } else {+ return ".";+ }+}++string+dirname(const string_view path) noexcept+{+ size_t e = path.size();+ if (e--) {+ for (; path[e] == '/'; e--)+ if (!e)+ return "/";+ for (; path[e] != '/'; e--)+ if (!e)+ return ".";+ for (; path[e] == '/'; e--)+ if (!e)+ return "/";+ return path.substr(0, e + 1);+ }+ return ".";+}++string+resolve(const string_view lhs, const string_view rhs) noexcept+{+ if (lhs.empty())+ return rhs;+ if (!rhs.empty() && rhs[0] == '/')+ return rhs;+ if (!lhs.empty() && lhs[lhs.size() - 1] == '/')+ return string(lhs) + rhs;+ return string(lhs) + "/" + rhs;+}diff --git a/llamafile/server/path.h b/llamafile/server/path.hnew file mode 100644index 00000000..c24822f5--- /dev/null+++ b/llamafile/server/path.h@@ -0,0 +1,28 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <ctl/string.h>++ctl::string+dirname(const ctl::string_view) noexcept;++ctl::string+basename(const ctl::string_view) noexcept;++ctl::string+resolve(const ctl::string_view, const ctl::string_view) noexcept;diff --git a/llamafile/server/server.cpp b/llamafile/server/server.cppnew file mode 100644index 00000000..3d3e8d98--- /dev/null+++ b/llamafile/server/server.cpp@@ -0,0 +1,193 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "server.h"++#include <assert.h>+#include <netinet/in.h>+#include <netinet/tcp.h>+#include <stdio.h>+#include <sys/auxv.h>+#include <sys/socket.h>+#include <time.h>++#include "llamafile/llamafile.h"+#include "log.h"+#include "server.h"+#include "worker.h"++Server::Server(int fd) : fd(fd)+{+}++Server::~Server()+{+ unassert(fd == -1);+ unassert(!worker_count.load(std::memory_order_relaxed));+ unassert(dll_is_empty(active_workers));+ unassert(dll_is_empty(idle_workers));+ pthread_mutex_destroy(&lock_);+ pthread_cond_destroy(&cond_);+}++void+Server::lock()+{+ pthread_mutex_lock(&lock_);+}++void+Server::signal()+{+ pthread_cond_signal(&cond_);+}++void+Server::wait()+{+ struct timespec waitfor =+ timespec_add(timespec_real(), timespec_fromseconds(1));+ pthread_cond_timedwait(&cond_, &lock_, &waitfor);+}++void+Server::unlock()+{+ pthread_mutex_unlock(&lock_);+}++void+Server::terminate()+{+ terminated.store(true, std::memory_order_release);+ signal();+}++int+Server::close()+{+ int rc = 0;+ if (fd != -1) {+ rc = ::close(fd);+ fd = -1;+ }+ return rc;+}++void*+worker_thread(void* arg)+{+ Worker* worker = (Worker*)arg;+ worker->run();+ return 0;+}++errno_t+Server::spawn()+{+ errno_t err;+ Worker* worker;+ pthread_attr_t attr;+ worker = new Worker(this);+ pthread_attr_init(&attr);+ pthread_attr_setstacksize(&attr, 65536);+ pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));+ pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);+ if ((err = pthread_create(&worker->th, &attr, worker_thread, worker)))+ delete worker;+ pthread_attr_destroy(&attr);+ return err;+}++int+Server::accept()+{+ // accept connection+ sockaddr_in clientaddr;+ set_thread_name("listen");+ uint32_t clientsize = sizeof(clientaddr);+ int clifd = ::accept(fd, (sockaddr*)&clientaddr, &clientsize);+ if (clifd == -1)+ return -1;++ // set name+ char name[17];+ unsigned ip = ntohl(clientaddr.sin_addr.s_addr);+ snprintf(name,+ sizeof(name),+ "%hhu.%hhu.%hhu.%hhu",+ ip >> 24,+ ip >> 16,+ ip >> 8,+ ip);+ set_thread_name(name);++ // keep sockets open+ if (FLAG_keepalive > 0) {+ int yes = 1;+ int secs = FLAG_keepalive;+ setsockopt(clifd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes));+ setsockopt(clifd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes));+ setsockopt(clifd, IPPROTO_TCP, TCP_KEEPIDLE, &secs, sizeof(secs));+ setsockopt(clifd, IPPROTO_TCP, TCP_KEEPINTVL, &secs, sizeof(secs));+ }++ if (FLAG_verbose >= 2)+ LOG("accept");+ return clifd;+}++void+Server::run()+{+ while (!terminated.load(std::memory_order_acquire)) {+ lock();+ if (!terminated.load(std::memory_order_acquire))+ wait();+ unlock();+ if (terminated.load(std::memory_order_acquire))+ break;+ int missing =+ FLAG_threads - worker_count.load(std::memory_order_acquire);+ for (int i = 0; i < missing; ++i)+ spawn();+ }+}++void+Server::shutdown()+{+ // on windows this is the only way accept() can be canceled+ if (IsWindows())+ close();++ // kill workers+ lock();+ for (Dll* e = dll_first(idle_workers); e; e = dll_next(idle_workers, e))+ WORKER(e)->kill();+ for (Dll* e = dll_first(active_workers); e; e = dll_next(active_workers, e))+ WORKER(e)->kill();+ unlock();++ // wait for workers to die+ while (worker_count.load(std::memory_order_acquire) > 0) {+ lock();+ if (worker_count.load(std::memory_order_acquire) > 0)+ wait();+ unlock();+ }+}diff --git a/llamafile/server/server.h b/llamafile/server/server.hnew file mode 100644index 00000000..967909fd--- /dev/null+++ b/llamafile/server/server.h@@ -0,0 +1,51 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <atomic>+#include <cosmo.h>+#include <pthread.h>++struct Server+{+ Server(int);+ ~Server();++ int accept();+ errno_t spawn();+ void terminate();+ void shutdown();+ int close();+ void run();+ void lock();+ void unlock();+ void signal();+ void wait();++ int fd;+ Dll* idle_workers = nullptr;+ Dll* active_workers = nullptr;+ pthread_cond_t cond_ = PTHREAD_COND_INITIALIZER;+ pthread_mutex_t lock_ = PTHREAD_MUTEX_INITIALIZER;+ std::atomic_int worker_count = ATOMIC_VAR_INIT(0);+ std::atomic_bool terminated = ATOMIC_VAR_INIT(false);+};++extern Server* g_server;++int+create_listening_socket(const char*);diff --git a/llamafile/server/signals.cpp b/llamafile/server/signals.cppnew file mode 100644index 00000000..f363ce5b--- /dev/null+++ b/llamafile/server/signals.cpp@@ -0,0 +1,94 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "signals.h"++#include <ucontext.h>++#include "log.h"+#include "server.h"++void+on_termination_signal(int sig)+{+ LOG("%G", sig);+ g_server->terminate();+}++void+on_crash_signal(int sig, siginfo_t* si, void* arg)+{+ LOG("crashed %G", sig);+ char message[256];+ describe_crash(message, sizeof(message), sig, si, arg);+ LOG("crashed %s", message);+ pthread_exit(PTHREAD_CANCELED);+}++void+setup_signals(void)+{+ struct sigaction sa;+ sa.sa_flags = SA_SIGINFO;+ sigemptyset(&sa.sa_mask);+ sa.sa_handler = on_termination_signal;++ sigaction(SIGINT, &sa, 0); // ctrl-c+ sigaction(SIGHUP, &sa, 0); // terminal close+ sigaction(SIGTERM, &sa, 0); // kill++ sa.sa_sigaction = on_crash_signal;+ sigaddset(&sa.sa_mask, SIGABRT); // abort()+ sigaddset(&sa.sa_mask, SIGTRAP); // breakpoint+ sigaddset(&sa.sa_mask, SIGFPE); // illegal math+ sigaddset(&sa.sa_mask, SIGBUS); // illegal memory+ sigaddset(&sa.sa_mask, SIGSEGV); // illegal memory+ sigaddset(&sa.sa_mask, SIGILL); // illegal instruction+ sigaddset(&sa.sa_mask, SIGXCPU); // out of cpu quota+ sigaddset(&sa.sa_mask, SIGXFSZ); // file too large++ sigaction(SIGABRT, &sa, 0); // abort()+ sigaction(SIGTRAP, &sa, 0); // breakpoint+ sigaction(SIGFPE, &sa, 0); // illegal math+ sigaction(SIGBUS, &sa, 0); // illegal memory+ sigaction(SIGSEGV, &sa, 0); // illegal memory+ sigaction(SIGILL, &sa, 0); // illegal instruction+ sigaction(SIGXCPU, &sa, 0); // out of cpu quota+ sigaction(SIGXFSZ, &sa, 0); // file too large+}++void+restore_signals(void)+{+ struct sigaction sa;+ sa.sa_flags = 0;+ sa.sa_handler = SIG_DFL;+ sigemptyset(&sa.sa_mask);++ sigaction(SIGINT, &sa, 0); // ctrl-c+ sigaction(SIGHUP, &sa, 0); // terminal close+ sigaction(SIGTERM, &sa, 0); // kill++ sigaction(SIGABRT, &sa, 0); // abort()+ sigaction(SIGTRAP, &sa, 0); // breakpoint+ sigaction(SIGFPE, &sa, 0); // illegal math+ sigaction(SIGBUS, &sa, 0); // illegal memory+ sigaction(SIGSEGV, &sa, 0); // illegal memory+ sigaction(SIGILL, &sa, 0); // illegal instruction+ sigaction(SIGXCPU, &sa, 0); // out of cpu quota+ sigaction(SIGXFSZ, &sa, 0); // file too large+}diff --git a/llamafile/server/signals.h b/llamafile/server/signals.hnew file mode 100644index 00000000..ab9104fe--- /dev/null+++ b/llamafile/server/signals.h@@ -0,0 +1,28 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <signal.h>++void+setup_signals(void);++void+restore_signals(void);++void+describe_crash(char*, size_t, int, siginfo_t*, void*);diff --git a/llamafile/server/time.cpp b/llamafile/server/time.cppnew file mode 100644index 00000000..c733f80f--- /dev/null+++ b/llamafile/server/time.cpp@@ -0,0 +1,203 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "time.h"++#include <atomic>+#include <pthread.h>+#include <signal.h>+#include <sys/auxv.h>+#include <unistd.h>++#include "log.h"++//+// lockless implementation of gmtime_r() and localtime_r()+//++struct Clock+{+ std::atomic_uint roll;+ std::atomic_ulong time;+ std::atomic_ulong date;+};++static Clock g_clck[2];++static void+set_clck(Clock* clck, long time, long date)+{+ unsigned long roll;+ roll = clck->roll.fetch_add(1, std::memory_order_relaxed);+ time &= 0xffffffffffff;+ date &= 0xffffffffffff;+ time |= roll << 48;+ date |= roll << 48;+ clck->time.store(time, std::memory_order_relaxed);+ clck->date.store(date, std::memory_order_relaxed);+}++static void+get_clck(Clock* clck, long* out_time, long* out_date)+{+ long time, date;+ do {+ time = clck->time.load(std::memory_order_relaxed);+ date = clck->date.load(std::memory_order_relaxed);+ } while ((time >> 48) != (date >> 48));+ *out_date = date & 0xffffffffffff;+ *out_time = time & 0xffffffffffff;+}++static long+encode_date(const tm* tm)+{+ long date;+ date = tm->tm_year;+ date <<= 4;+ date |= tm->tm_isdst == 1;+ date <<= 1;+ date |= tm->tm_mon;+ date <<= 5;+ date |= tm->tm_mday;+ date <<= 3;+ date |= tm->tm_wday;+ date <<= 5;+ date |= tm->tm_hour;+ date <<= 6;+ date |= tm->tm_min;+ date <<= 6;+ date |= tm->tm_sec;+ return date;+}++static void+decode_date(long date, tm* tm)+{+ tm->tm_sec = date & 63;+ date >>= 6;+ tm->tm_min = date & 63;+ date >>= 6;+ tm->tm_hour = date & 31;+ date >>= 5;+ tm->tm_wday = date & 7;+ date >>= 3;+ tm->tm_mday = date & 31;+ date >>= 5;+ tm->tm_mon = date & 15;+ date >>= 4;+ tm->tm_isdst = date & 1;+ date >>= 1;+ tm->tm_year = date;+ tm->tm_gmtoff = 0; // unsupported+ tm->tm_zone = 0; // unsupported+ tm->tm_yday = 0; // unsupported+}++static void+update_time()+{+ tm tm;+ timespec ts;+ clock_gettime(0, &ts);+ gmtime_r(&ts.tv_sec, &tm);+ set_clck(&g_clck[0], ts.tv_sec, encode_date(&tm));+ localtime_r(&ts.tv_sec, &tm);+ set_clck(&g_clck[1], ts.tv_sec, encode_date(&tm));+}++static void*+time_worker(void* arg)+{+ sigset_t ss;+ sigemptyset(&ss);+ sigaddset(&ss, SIGHUP);+ sigaddset(&ss, SIGINT);+ sigaddset(&ss, SIGQUIT);+ sigaddset(&ss, SIGTERM);+ sigaddset(&ss, SIGUSR1);+ sigaddset(&ss, SIGALRM);+ pthread_sigmask(SIG_BLOCK, &ss, 0);+ set_thread_name("localtime");+ for (;;) {+ sleep(10);+ update_time();+ }+ return nullptr;+}++void+time_init()+{+ update_time();+ pthread_t th;+ pthread_attr_t attr;+ pthread_attr_init(&attr);+ pthread_attr_setstacksize(&attr, 65536);+ pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));+ pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);+ if (pthread_create(&th, &attr, time_worker, 0))+ __builtin_trap();+ pthread_attr_destroy(&attr);+}++static const char kMonDays[2][12] = {+ { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },+ { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },+};++static void+time_lockless(Clock* clck, long now, tm* tm)+{+ long time, date, since;+ get_clck(clck, &time, &date);+ decode_date(date, tm);+ since = now - time;+ since = since < 60 ? since : 60;+ for (; since > 0; --since) {+ if (++tm->tm_sec >= 60) {+ tm->tm_sec = 0;+ if (++tm->tm_min >= 60) {+ tm->tm_min = 0;+ if (++tm->tm_hour >= 24) {+ tm->tm_hour = 0;+ if (++tm->tm_mday >= 7)+ tm->tm_mday = 0;+ if (++tm->tm_mday > kMonDays[!!tm->tm_isdst][tm->tm_mon]) {+ tm->tm_mday = 1;+ if (++tm->tm_mon >= 12) {+ tm->tm_mon = 0;+ ++tm->tm_year;+ }+ }+ }+ }+ }+ }+}++void+gmtime_lockless(long now, tm* tm)+{+ time_lockless(&g_clck[0], now, tm);+}++void+localtime_lockless(long now, tm* tm)+{+ time_lockless(&g_clck[1], now, tm);+}diff --git a/llamafile/security.c b/llamafile/server/time.hsimilarity index 71%rename from llamafile/security.crename to llamafile/server/time.hindex 84781bff..3ebe77e7 100644--- a/llamafile/security.c+++ b/llamafile/server/time.h@@ -1,5 +1,5 @@-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi//// Copyright 2024 Mozilla Foundation//@@ -15,6 +15,14 @@// See the License for the specific language governing permissions and// limitations under the License.-#include "llamafile.h"+#pragma once+#include <time.h>-bool FLAG_unsecure;+void+time_init();++void+gmtime_lockless(long, tm*);++void+localtime_lockless(long, tm*);diff --git a/llamafile/server/utils.h b/llamafile/server/utils.hnew file mode 100644index 00000000..0a5c3940--- /dev/null+++ b/llamafile/server/utils.h@@ -0,0 +1,27 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <ctl/string_view.h>++extern const signed char kHexToInt[256];++bool+atob(ctl::string_view, bool);++char*+hexcpy(char*, unsigned long);diff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cppnew file mode 100644index 00000000..e9746501--- /dev/null+++ b/llamafile/server/worker.cpp@@ -0,0 +1,140 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "worker.h"++#include <assert.h>+#include <exception>+#include <pthread.h>++#include "client.h"+#include "llamafile/llamafile.h"+#include "log.h"+#include "signals.h"++Worker::Worker(Server* server) : server(server)+{+ dll_init(&elem);+}++void+Worker::kill()+{+ pthread_cancel(th);+}++void+Worker::begin()+{+ unassert(!working);+ server->lock();+ dll_remove(&server->idle_workers, &elem);+ if (dll_is_empty(server->idle_workers)) {+ Dll* slowbro;+ if ((slowbro = dll_last(server->active_workers))) {+ LOG("all threads active! dropping oldest client");+ WORKER(slowbro)->kill();+ }+ }+ working = true;+ dll_make_first(&server->active_workers, &elem);+ server->unlock();+}++void+Worker::end()+{+ unassert(working);+ server->lock();+ dll_remove(&server->active_workers, &elem);+ working = false;+ dll_make_first(&server->idle_workers, &elem);+ server->unlock();+}++void+Worker::retire()+{+ server->lock();+ if (working)+ dll_remove(&server->active_workers, &elem);+ else+ dll_remove(&server->idle_workers, &elem);+ server->worker_count.fetch_sub(1, std::memory_order_acq_rel);+ server->signal();+ server->unlock();+ delete this;+}++void+Worker::handle(void)+{+ if ((client.fd = server->accept()) == -1) {+ LOG("accept returned %m");+ return;+ }++ begin();+ pthread_cleanup_push(+ [](void* arg) {+ Worker* worker = (Worker*)arg;+ worker->client.close();+ worker->end();+ },+ this);++ try {+ client.run();+ } catch (const std::exception& e) {+ LOG("caught %s", e.what());+ } catch (...) {+ LOG("caught unknown exception");+ }++ pthread_cleanup_pop(true);+}++void+Worker::run()+{+ server->lock();+ dll_make_first(&server->idle_workers, &elem);+ server->worker_count.fetch_add(1, std::memory_order_acq_rel);+ server->unlock();++ pthread_cleanup_push(+ [](void* arg) {+ Worker* worker = (Worker*)arg;+ worker->retire();+ },+ this);++ while (!server->terminated.load(std::memory_order_acquire)) {+ sigset_t mask;+ sigemptyset(&mask);+ sigaddset(&mask, SIGHUP);+ sigaddset(&mask, SIGINT);+ sigaddset(&mask, SIGQUIT);+ sigaddset(&mask, SIGTERM);+ sigaddset(&mask, SIGUSR1);+ sigaddset(&mask, SIGALRM);+ pthread_sigmask(SIG_BLOCK, &mask, 0);+ handle();+ }++ pthread_cleanup_pop(true);+}diff --git a/llamafile/server/worker.h b/llamafile/server/worker.hnew file mode 100644index 00000000..1b65bff4--- /dev/null+++ b/llamafile/server/worker.h@@ -0,0 +1,41 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include "client.h"+#include "server.h"+#include <cosmo.h>+#include <pthread.h>++#define WORKER(e) DLL_CONTAINER(Worker, elem, e)++struct Worker+{+ Server* server;+ Dll elem;+ pthread_t th = 0;+ bool working = false;+ Client client;++ Worker(Server*);+ void run();+ void begin();+ void handle(void);+ void end();+ void retire();+ void kill();+};diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex 33fcd603..ca0606f9 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -11,14 +11,14 @@ bool iqk_mul_mat(long, long, long, int, const void *, const void *, float *, lonbool iqk_mul_mat_zen4(long, long, long, int, const void *, const void *, float *, long, int, int);bool iqk_mul_mat_arm82(long, long, long, int, const void *, const void *, float *, long, int, int);-bool iqk_mul_mat_moe(long, long, long, int, int, const void *, const void *,- float *, long, long, const void *, int, int);-bool iqk_mul_mat_moe_zen4(long, long, long, int, int, const void *, const void *,- float *, long, long, const void *, int, int);-bool iqk_mul_mat_moe_arm82(long, long, long, int, int, const void *, const void *,- float *, long, long, const void *, int, int);-bool iqk_mul_mat_moe_unsupported(long, long, long, int, int, const void *, const void *,- float *, long, long, const void *, int, int);+bool iqk_mul_mat_moe(long, long, long, int, int, const void *, const void *, float *, long, long,+ const void *, int, int);+bool iqk_mul_mat_moe_zen4(long, long, long, int, int, const void *, const void *, float *, long,+ long, const void *, int, int);+bool iqk_mul_mat_moe_arm82(long, long, long, int, int, const void *, const void *, float *, long,+ long, const void *, int, int);+bool iqk_mul_mat_moe_unsupported(long, long, long, int, int, const void *, const void *, float *,+ long, long, const void *, int, int);bool llamafile_sgemm(long, long, long, const void *, long, const void *, long, void *, long, int,int, int, int, int, int, int);@@ -73,8 +73,8 @@ bool llamafile_mixmul_arm80(const struct ggml_compute_params *, const struct ggmbool llamafile_mixmul_arm82(const struct ggml_compute_params *, const struct ggml_tensor *,const struct ggml_tensor *, const struct ggml_tensor *,struct ggml_tensor *);-bool llamafile_mixmul_iqk(long, long, long, int, int, const void *, const void *,- float *, long, long, const void *, int, int);+bool llamafile_mixmul_iqk(long, long, long, int, int, const void *, const void *, float *, long,+ long, const void *, int, int);#ifdef __cplusplus}diff --git a/llamafile/tinyblas_cpu_unsupported.cpp b/llamafile/tinyblas_cpu_unsupported.cppindex 166494a4..8347ef3b 100644--- a/llamafile/tinyblas_cpu_unsupported.cpp+++ b/llamafile/tinyblas_cpu_unsupported.cpp@@ -17,9 +17,9 @@#include "sgemm.h"-bool llamafile_sgemm_unsupported(int m, int n, int k, const void *A, int lda, const void *B,- int ldb, void *C, int ldc, int ith, int nth, int task, int Atype,- int Btype, int Ctype) {+bool llamafile_sgemm_unsupported(long m, long n, long k, const void *A, long lda, const void *B,+ long ldb, void *C, long ldc, int ith, int nth, int task, int Atype,+ int Btype, int Ctype, int precision) {return false;}@@ -30,7 +30,7 @@ bool llamafile_mixmul_unsupported(const struct ggml_compute_params *params,return false;}-bool iqk_mul_mat_moe_unsupported(long, long, long, int, int, const void *, const void *,- float *, long, long, const void *, int, int) {+bool iqk_mul_mat_moe_unsupported(long, long, long, int, int, const void *, const void *, float *,+ long, long, const void *, int, int) {return false;}diff --git a/llamafile/tokenize.cpp b/llamafile/tokenize.cppindex ab135fc8..facc42c6 100644--- a/llamafile/tokenize.cpp+++ b/llamafile/tokenize.cpp@@ -15,68 +15,82 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <cerrno>-#include <cmath>-#include <cosmo.h>-#include <cstdio>-#include <cstring>-#include <vector>+#include "llamafile.h"+#include "version.h"++#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <tool/args/args.h>-#include "llama.cpp/common.h"#include "llama.cpp/llama.h"-#include "llamafile/llamafile.h"int main(int argc, char **argv) {+ llamafile_check_cpu();if (llamafile_has(argv, "--version")) {puts("llamafile-tokenize v" LLAMAFILE_VERSION_STRING);return 0;}- llamafile_check_cpu();- log_disable();-- gpt_params params;- params.n_ctx = 0;+ FLAG_log_disable = true;- if (!gpt_params_parse(argc, argv, params))- return 1;+ LoadZipArgs(&argc, &argv);+ llamafile_get_flags(argc, argv);- llama_model_params model_params = llama_model_default_params();- llama_model *model = llama_load_model_from_file(params.model.c_str(), model_params);+ llama_model_params mparams = {+ .n_gpu_layers = 0,+ .split_mode = (enum llama_split_mode)FLAG_split_mode,+ .main_gpu = 0,+ .tensor_split = nullptr,+ .rpc_servers = nullptr,+ .progress_callback = nullptr,+ .progress_callback_user_data = nullptr,+ .kv_overrides = nullptr,+ .vocab_only = true,+ .use_mmap = true,+ .use_mlock = false,+ .check_tensors = false,+ };+ llama_model *model = llama_load_model_from_file(FLAG_model, mparams);if (model == NULL)return 3;- llama_context_params ctx_params = llama_context_params_from_gpt_params(params);- llama_context *ctx = llama_new_context_with_model(model, ctx_params);- if (ctx == NULL)- return 4;-- bool should_read_stdin = params.prompt.empty();+ FILE *input;+ if (FLAG_prompt) {+ input = fmemopen((void *)FLAG_prompt, strlen(FLAG_prompt), "rb");+ } else if (FLAG_file) {+ if (!(input = fopen(FLAG_file, "rb"))) {+ perror(FLAG_file);+ exit(1);+ }+ } else {+ input = stdin;+ }for (;;) {- ssize_t n;- char buf[4097];- const char *input;- if (should_read_stdin) {- n = read(0, buf, 4096);- if (n == -1) {- fprintf(stderr, "/dev/stdin: %s\n", strerror(errno));+ char *text;+ size_t textlen;+ if (!(text = fgetln(input, &textlen)))+ break;++ static llama_token toks[4096];+ int count = llama_tokenize(model, text, textlen, toks, 4096, false, false);+ if (count < 0) {+ fprintf(stderr, "%s: failed to tokenize line\n", argv[0]);+ exit(1);+ }++ for (int i = 0; i < count; ++i) {++ char s[256];+ int n = llama_token_to_piece(model, toks[i], s, sizeof(s), false);+ if (n < 0) {+ fprintf(stderr, "%s: failed to convert token %d to string\n", argv[0], toks[i]);exit(1);}- if (!n)- break;- buf[n] = 0;- input = buf;- } else {- input = params.prompt.c_str();- }- std::vector<llama_token> toks = ::llama_tokenize(ctx, input, false);- for (llama_token tok : toks) {- std::string str = llama_token_to_piece(ctx, tok, true);- const char *s = str.c_str();- for (int i = 0; s[i]; ++i) {+ for (int i = 0; i < n; ++i) {int c = s[i] & 255;switch (c) {case '\\':@@ -123,6 +137,5 @@ int main(int argc, char **argv) {}}- llama_free(ctx);llama_free_model(model);}diff --git a/llamafile/zipalign.c b/llamafile/zipalign.cindex c43a24ca..5fdf3304 100644--- a/llamafile/zipalign.c+++ b/llamafile/zipalign.c@@ -17,10 +17,10 @@#include "llamafile.h"#include "zip.h"+#include <assert.h>#include <cosmo.h>#include <fcntl.h>-#include <getopt.h>#include <libgen.h>#include <limits.h>#include <stdbool.h>@@ -29,9 +29,13 @@#include <stdlib.h>#include <sys/stat.h>#include <sys/uio.h>+#include <third_party/getopt/getopt.internal.h>#include <third_party/zlib/zlib.h>#include <time.h>+#define TINYMALLOC_MAX_BYTES (64 * 1024 * 1024)+#include <libc/mem/tinymalloc.inc>+#define CHUNK 2097152#define Min(a, b) ((a) < (b) ? (a) : (b))@@ -39,11 +43,11 @@#define DOS_TIME(HOUR, MINUTE, SECOND) ((HOUR) << 11 | (MINUTE) << 5 | (SECOND) >> 1)static const char *prog;-static int FLAG_junk;-static int FLAG_level;-static int FLAG_verbose;-static int FLAG_alignment = 65536;-static bool FLAG_nondeterministic;+static int flag_junk;+static int flag_level;+static int flag_verbose;+static int flag_alignment = 65536;+static bool flag_nondeterministic;static wontreturn void Die(const char *thing, const char *reason) {tinyprint(2, thing, ": fatal error: ", reason, "\n", NULL);@@ -113,23 +117,23 @@ int main(int argc, char *argv[]) {case '7':case '8':case '9':- FLAG_level = opt - '0';+ flag_level = opt - '0';break;case 'v':- ++FLAG_verbose;+ ++flag_verbose;break;case 'j':- FLAG_junk = true;+ flag_junk = true;break;case 'N':- FLAG_nondeterministic = true;+ flag_nondeterministic = true;break;case 'a':- FLAG_alignment = atoi(optarg);- if (FLAG_alignment < 1)- Die(prog, "FLAG_alignment must be at least 1");- if (FLAG_alignment & (FLAG_alignment - 1))- Die(prog, "FLAG_alignment must be two power");+ flag_alignment = atoi(optarg);+ if (flag_alignment < 1)+ Die(prog, "flag_alignment must be at least 1");+ if (flag_alignment & (flag_alignment - 1))+ Die(prog, "flag_alignment must be two power");break;default:return 1;@@ -201,7 +205,7 @@ int main(int argc, char *argv[]) {char **names = Malloc(sizeof(char *) * argc);for (int i = optind; i < argc; ++i) {names[i] = StrDup(argv[i]);- if (FLAG_junk)+ if (flag_junk)names[i] = basename(names[i]);elsewhile (*names[i] == '/')@@ -266,7 +270,7 @@ int main(int argc, char *argv[]) {// get timeint64_t ts;uint16_t mtime, mdate;- if (FLAG_nondeterministic)+ if (flag_nondeterministic)ts = st.st_mtime;elsets = 1700000000;@@ -277,20 +281,20 @@ int main(int argc, char *argv[]) {size_t namlen = strlen(name);size_t extlen = (2 + 2 + 8 + 8);size_t hdrlen = kZipLfileHdrMinSize + namlen + extlen;- while ((zsize + hdrlen) & (FLAG_alignment - 1))+ while ((zsize + hdrlen) & (flag_alignment - 1))++zsize;// initialize zlib in raw deflate modez_stream zs;int compression;- if (!FLAG_level) {+ if (!flag_level) {compression = kZipCompressionNone;} else {compression = kZipCompressionDeflate;zs.zalloc = 0;zs.zfree = 0;zs.opaque = 0;- switch (deflateInit2(&zs, FLAG_level, Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,+ switch (deflateInit2(&zs, flag_level, Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,Z_DEFAULT_STRATEGY)) {case Z_OK:break;@@ -312,7 +316,7 @@ int main(int argc, char *argv[]) {if ((rc = pread(fd, iobuf, Min(size, CHUNK), i)) <= 0)DieSys(path);crc = crc32(crc, iobuf, rc);- if (!FLAG_level) {+ if (!flag_level) {// write uncompressed chunk to outputif (pwrite(zfd, iobuf, rc, zsize + hdrlen + compsize) != rc)DieSys(zpath);@@ -340,7 +344,7 @@ int main(int argc, char *argv[]) {} while (!zs.avail_out);}}- if (FLAG_level)+ if (flag_level)unassert(deflateEnd(&zs) == Z_OK);// write local file header@@ -411,7 +415,7 @@ int main(int argc, char *argv[]) {DieSys(path);// log asset creation- if (FLAG_verbose)+ if (flag_verbose)tinyprint(2, path, " -> ", name, "\n", NULL);}