Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add seccomp, tokenbucket, and batch prioritization
Justine Tunney99dd1c01084e
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 86dae40a..af936585 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -17,6 +17,8 @@ LLAMAFILE_OBJS := \$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)+$(LLAMAFILE_OBJS): private CCFLAGS += -g+# this executable defines its own malloc(), free(), etc.# therefore we want to avoid it going inside the .a fileLLAMAFILE_OBJS := $(filter-out o/$(MODE)/llamafile/zipalign.o,$(LLAMAFILE_OBJS))@@ -52,6 +54,7 @@ o/$(MODE)/llamafile: \o/$(MODE)/llamafile/addnl \o/$(MODE)/llamafile/pool_test.runs \o/$(MODE)/llamafile/pool_cancel_test.runs \+ o/$(MODE)/llamafile/parse_cidr_test.runs \################################################################################# microarchitectures@@ -143,6 +146,11 @@ o/$(MODE)/llamafile/tinyblas_cpu_sgemm_arm82.o: \################################################################################# testing+o/$(MODE)/llamafile/parse_cidr_test: \+ o/$(MODE)/llamafile/parse_cidr_test.o \+ o/$(MODE)/llamafile/parse_cidr.o \+ o/$(MODE)/llamafile/parse_ip.o \+o/$(MODE)/llamafile/pool_test: \o/$(MODE)/llamafile/pool_test.o \o/$(MODE)/llamafile/crash.o \diff --git a/llamafile/flags.cpp b/llamafile/flags.cppindex 761ccf52..d2f0a703 100644--- a/llamafile/flags.cpp+++ b/llamafile/flags.cpp@@ -17,11 +17,15 @@#include "debug.h"#include "llamafile.h"+#include "trust.h"#include <errno.h>#include <fcntl.h>+#include <limits.h>+#include <math.h>#include <stdio.h>#include <stdlib.h>+#include <time.h>#include <unistd.h>#include "llama.cpp/llama.h"@@ -38,9 +42,11 @@ bool FLAG_tinyblas = false;bool FLAG_trace = false;bool FLAG_unsecure = false;const char *FLAG_file = nullptr;+const char *FLAG_ip_header = nullptr;const char *FLAG_listen = "0.0.0.0:8080";const char *FLAG_model = nullptr;const char *FLAG_prompt = nullptr;+double FLAG_token_rate = 1;float FLAG_temp = 0.8;int FLAG_batch = 2048;int FLAG_ctx = 512;@@ -54,6 +60,8 @@ int FLAG_n_gpu_layers = -1;int FLAG_seed = LLAMA_DEFAULT_SEED;int FLAG_split_mode = LLAMA_SPLIT_MODE_LAYER;int FLAG_threads;+int FLAG_token_burst = 100;+int FLAG_token_cidr = 24;int FLAG_ubatch = 512;int FLAG_verbose = 0;int FLAG_warmup = true;@@ -136,6 +144,61 @@ void llamafile_get_flags(int argc, char **argv) {continue;}+ if (!strcmp(flag, "--ip-header")) {+ if (i == argc)+ missing("--ip-header");+ cidr ip;+ FLAG_ip_header = argv[i++];+ continue;+ }++ if (!strcmp(flag, "--trust")) {+ if (i == argc)+ missing("--trust");+ cidr ip;+ if (!parse_cidr(argv[i++], &ip))+ error("invalid --trust CIDR (expect like 192.168.0.0/24)");+ FLAG_trust.push_back(ip);+ continue;+ }++ if (!strcmp(flag, "--token-rate")) {+ if (i == argc)+ missing("--token-rate");+ double rate = atof(argv[i++]);+ if (!rate)+ error("--token-rate can't be zero");+ double micros = 1e6 / rate;+ if (isnan(micros) || isinf(micros))+ error("--token-rate invalid");+ if (micros < 10)+ error("--token-rate too frequent (can't be above 100000 a.k.a. 1 per 10µs)");+ if (micros > 60 * 60 * 1e6)+ error("--token-rate too infrequent (can't be below 1/(60*60) a.k.a. 1 per hour)");+ FLAG_token_rate = rate;+ continue;+ }++ if (!strcmp(flag, "--token-burst")) {+ if (i == argc)+ missing("--token-burst");+ int burst = atoi(argv[i++]);+ if (!(1 <= burst && burst <= 127))+ error("--token-burst must be 1..127");+ FLAG_token_burst = burst;+ continue;+ }++ if (!strcmp(flag, "--token-cidr")) {+ if (i == argc)+ missing("--token-cidr");+ int cidr = atoi(argv[i++]);+ if (!(3 <= cidr && cidr <= 32))+ error("--token-cidr must be 3..32");+ FLAG_token_cidr = cidr;+ continue;+ }+//////////////////////////////////////////////////////////////////////// http server flagsdiff --git a/llamafile/is_loopback_ip.cpp b/llamafile/is_loopback_ip.cppnew file mode 100644index 00000000..314e9d06--- /dev/null+++ b/llamafile/is_loopback_ip.cpp@@ -0,0 +1,22 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "trust.h"++bool is_loopback_ip(unsigned x) noexcept {+ return (x >> 24) == 127; /* 127.0.0.0/8 */+}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 15b0bc20..bc1cf090 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -19,9 +19,11 @@ extern bool FLAG_trace;extern bool FLAG_trap;extern bool FLAG_unsecure;extern const char *FLAG_file;+extern const char *FLAG_ip_header;extern const char *FLAG_listen;extern const char *FLAG_model;extern const char *FLAG_prompt;+extern double FLAG_token_rate;extern float FLAG_temp;extern int FLAG_batch;extern int FLAG_ctx;@@ -36,6 +38,8 @@ extern int FLAG_n_gpu_layers;extern int FLAG_seed;extern int FLAG_split_mode;extern int FLAG_threads;+extern int FLAG_token_burst;+extern int FLAG_token_cidr;extern int FLAG_ubatch;extern int FLAG_verbose;extern int FLAG_warmup;diff --git a/llamafile/parse_cidr.cpp b/llamafile/parse_cidr.cppnew file mode 100644index 00000000..f2b08bf6--- /dev/null+++ b/llamafile/parse_cidr.cpp@@ -0,0 +1,54 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "trust.h"++#include <ctype.h>+#include <string.h>++bool parse_cidr(const ctl::string_view &str, cidr *out_cidr) noexcept {+ long ip;+ int bits;+ size_t i, n;+ const char *s, *p;+ s = str.data();+ n = str.size();+ if ((p = (const char *)memchr(s, '/', n))) {+ if ((ip = parse_ip(ctl::string_view(s, (i = p - s)))) == -1)+ return false;+ bits = 0;+ for (++i; i < n; ++i) {+ if (!isdigit(s[i]))+ return false;+ bits *= 10;+ bits += s[i] - '0';+ if (bits > 32)+ return false;+ }+ if (bits <= 0)+ return false;+ } else {+ if ((ip = parse_ip(str)) == -1)+ return false;+ bits = 32;+ }+ if (out_cidr) {+ out_cidr->ip = ip;+ out_cidr->bits = bits;+ }+ return true;+}diff --git a/llamafile/parse_cidr_test.cpp b/llamafile/parse_cidr_test.cppnew file mode 100644index 00000000..a7a5bd00--- /dev/null+++ b/llamafile/parse_cidr_test.cpp@@ -0,0 +1,66 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "trust.h"++int main(int argc, char *argv[]) {+ cidr in;++ if (!parse_cidr("10.10.10.1/24", &in))+ return 1;+ if (in.ip != 0x0a0a0a01)+ return 2;+ if (in.bits != 24)+ return 3;+ if (in.matches(parse_ip("10.10.11.1")))+ return 3;+ if (!in.matches(parse_ip("10.10.10.1")))+ return 3;+ if (!in.matches(parse_ip(" 10.10.10.255 ")))+ return 3;++ if (!parse_cidr("168430081/1", &in))+ return 4;+ if (in.ip != 0x0a0a0a01)+ return 5;+ if (in.bits != 1)+ return 6;++ if (parse_cidr("", 0))+ return 77;+ if (parse_cidr(" ", 0))+ return 78;+ if (parse_cidr("10.10.10.1/", 0))+ return 7;+ if (parse_cidr("10.10.10.1/a", 0))+ return 8;+ if (parse_cidr("10.10.10.1/33", 0))+ return 9;+ if (parse_cidr("10.10.10.a", 0))+ return 10;++ if (!parse_cidr("10.10.10.255", &in))+ return 11;+ if (in.ip != 0x0a0a0aff)+ return 12;+ if (in.bits != 32)+ return 13;+ if (in.matches(parse_ip("10.10.10.254")))+ return 14;+ if (!in.matches(parse_ip("10.10.10.255")))+ return 15;+}diff --git a/llamafile/parse_ip.cpp b/llamafile/parse_ip.cppnew file mode 100644index 00000000..da49db1e--- /dev/null+++ b/llamafile/parse_ip.cpp@@ -0,0 +1,71 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "trust.h"++#include <ctype.h>+#include <stdckdint.h>+#include <string.h>++long parse_ip(const ctl::string_view &str) noexcept {+ int c, j;+ size_t i, n;+ unsigned b, x;+ const char *s;+ bool dotted = false;+ s = str.data();+ n = str.size();++ // trim leading spaces+ while (n && isspace(*s)) {+ ++s;+ --n;+ }++ // trim trailing spaces+ while (n && isspace(s[n - 1]))+ --n;++ // forbid empty string+ if (!n)+ return -1;++ // parse ipv4 address+ for (b = x = j = i = 0; i < n; ++i) {+ c = s[i] & 255;+ if (isdigit(c)) {+ if (ckd_mul(&b, b, 10) || //+ ckd_add(&b, b, c - '0') || //+ (b > 255 && dotted)) {+ return -1;+ }+ } else if (c == '.') {+ if (b > 255)+ return -1;+ dotted = true;+ x <<= 8;+ x |= b;+ b = 0;+ ++j;+ } else {+ return -1;+ }+ }+ x <<= 8;+ x |= b;+ return x;+}diff --git a/llamafile/server/BUILD.mk b/llamafile/server/BUILD.mkindex c763de4f..37276db3 100644--- a/llamafile/server/BUILD.mk+++ b/llamafile/server/BUILD.mk@@ -9,6 +9,8 @@ LLAMAFILE_SERVER_INCS = $(filter %.inc,$(LLAMAFILE_SERVER_FILES))LLAMAFILE_SERVER_SRCS = $(filter %.cpp,$(LLAMAFILE_SERVER_FILES))LLAMAFILE_SERVER_OBJS = $(LLAMAFILE_SERVER_SRCS:%.cpp=o/$(MODE)/%.o)+$(LLAMAFILE_SERVER_OBJS): private CCFLAGS += -g+o/$(MODE)/llamafile/server/server.a: \$(filter-out %_test.o,$(LLAMAFILE_SERVER_OBJS))@@ -33,6 +35,12 @@ o/$(MODE)/llamafile/server/json_test: \o/$(MODE)/llamafile/server/hextoint.o \o/$(MODE)/double-conversion/double-conversion.a \+o/$(MODE)/llamafile/server/tokenbucket_test: \+ o/$(MODE)/llamafile/server/tokenbucket_test.o \+ o/$(MODE)/llamafile/server/tokenbucket.o \+ o/$(MODE)/llamafile/server/log.o \+ o/$(MODE)/llama.cpp/llama.cpp.a \+.PHONY: o/$(MODE)/llamafile/servero/$(MODE)/llamafile/server: \o/$(MODE)/llamafile/server/main \diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cppindex 546dba45..a28691ca 100644--- a/llamafile/server/client.cpp+++ b/llamafile/server/client.cpp@@ -28,10 +28,13 @@#include "llama.cpp/llama.h"#include "llamafile/llamafile.h"#include "llamafile/threadlocal.h"+#include "llamafile/trust.h"#include "llamafile/version.h"#include "log.h"#include "time.h"+#include "tokenbucket.h"+#include "worker.h"#define STANDARD_RESPONSE_HEADERS \"Server: llamafile/" LLAMAFILE_VERSION_STRING "\r\n" \@@ -188,6 +191,36 @@ Client::transport()return send_error(417);}+ effective_ip = client_ip;+ effective_ip_trusted = client_ip_trusted;+ if (FLAG_ip_header) {+ if (is_loopback_ip(client_ip) || client_ip_trusted) {+ ctl::string_view ip_header = get_header(FLAG_ip_header);+ if (!ip_header.empty()) {+ long ip;+ if ((ip = parse_ip(ip_header)) == -1) {+ effective_ip = ip;+ effective_ip_trusted = is_trusted_ip(ip);+ } else {+ SLOG("client's --ip-header wasn't a single ipv4 address");+ effective_ip_trusted = false;+ }+ }+ } else {+ SLOG("received direct connect from untrusted ip");+ effective_ip_trusted = false;+ }+ }++ if (get_header("X-Priority") == "batch") {+ worker->deprioritize();+ } else if (!effective_ip_trusted) {+ if (tokenbucket_acquire(client_ip) > FLAG_token_burst) {+ SLOG("deprioritizing");+ worker->deprioritize();+ }+ }+if (HasHeader(kHttpTransferEncoding))if (!HeaderEqualCase(kHttpTransferEncoding, "identity")) {close_connection = true;@@ -354,6 +387,28 @@ Client::has_at_most_this_element(int h, const string_view s)return true;}+ctl::string_view+Client::get_header(const ctl::string_view& key)+{+ int h;+ size_t i, keylen;+ if ((h = GetHttpHeader(key.data(), key.size())) != -1) {+ if (msg.headers[h].a)+ return ctl::string_view(ibuf.p + msg.headers[h].a,+ msg.headers[h].b - msg.headers[h].a);+ } else {+ for (i = 0; i < msg.xheaders.n; ++i)+ if (SlicesEqualCase(key.data(),+ key.size(),+ ibuf.p + msg.xheaders.p[i].k.a,+ msg.xheaders.p[i].k.b - msg.xheaders.p[i].k.a))+ return ctl::string_view(ibuf.p + msg.xheaders.p[i].v.a,+ msg.xheaders.p[i].v.b -+ msg.xheaders.p[i].v.a);+ }+ return ctl::string_view();+}+boolClient::read_payload(){diff --git a/llamafile/server/client.h b/llamafile/server/client.hindex 0b16f642..407dc349 100644--- a/llamafile/server/client.h+++ b/llamafile/server/client.h@@ -36,6 +36,7 @@#define HeaderEqualCase(H, S) \SlicesEqualCase(S, strlen(S), HeaderData(H), HeaderLength(H))+struct Worker;struct TokenizeParams;struct EmbeddingParams;struct CompletionParams;@@ -50,9 +51,14 @@ struct Cleanupstruct Client{int fd = -1;+ unsigned client_ip = 0;+ unsigned effective_ip = 0;+ bool client_ip_trusted = false;+ bool effective_ip_trusted = false;bool close_connection = false;bool should_send_error_if_canceled;size_t unread = 0;+ Worker* worker;timespec message_started;HttpMessage msg;Url url = {};@@ -84,6 +90,7 @@ struct Clientbool send2(const ctl::string_view, const ctl::string_view) __wur;char* append_header(const ctl::string_view, const ctl::string_view);bool has_at_most_this_element(int, const ctl::string_view);+ ctl::string_view get_header(const ctl::string_view&);ctl::string_view path();ctl::optional<ctl::string_view> param(ctl::string_view);diff --git a/llamafile/server/doc/technical_details.md b/llamafile/server/doc/technical_details.mdindex c8ca1a49..f58f9450 100644--- a/llamafile/server/doc/technical_details.md+++ b/llamafile/server/doc/technical_details.md@@ -9,8 +9,8 @@ technical aspects of LLaMAfiler.This server can serve embeddings an order of a magnitude faster than thellama.cpp example server. We measured this on Threadripper PRO 7995WX w/all-MiniLM-L6-v2 Q6\_K. Using a prompt with 50 tokens this server served-930 `/embedding` requests per second, whereas llama.cpp/build/bin/server-built with glibc served 100 per second.+1,916 `/embedding` requests per second, whereas+llama.cpp/build/bin/server built with glibc served 771 per second.The following wrk script was used:@@ -23,10 +23,10 @@ wrk.headers["Content-Type"] = "application/json"With the following commands:```-llamafiler -m /weights/all-MiniLM-L6-v2.Q6_K.gguf+llamafiler -m /weights/all-MiniLM-L6-v2.Q6_K.gguf --trust 127.0.0.1/32wrk --latency -t 32 -c 32 -s embedding.lua 'http://127.0.0.1:8080/embedding'-~/llama.cpp/build/bin/server -m /weights/all-MiniLM-L6-v2.Q6_K.gguf --embeddings+~/llama.cpp/llama-server -m /weights/all-MiniLM-L6-v2.Q6_K.gguf --embeddingswrk --latency -t 32 -c 32 -s embedding.lua 'http://127.0.0.1:8080/embedding'```@@ -34,39 +34,39 @@ The results for llamafiler were:```Thread Stats Avg Stdev Max +/- Stdev- Latency 33.89ms 7.48ms 61.55ms 66.90%- Req/Sec 29.26 6.52 50.00 59.74%+ Latency 16.50ms 3.01ms 31.95ms 71.14%+ Req/Sec 59.95 7.17 80.00 52.34%Latency Distribution- 50% 33.31ms- 75% 38.76ms- 90% 43.76ms- 99% 53.31ms- 1592 requests in 1.70s, 8.26MB read-Requests/sec: 934.80-Transfer/sec: 4.85MB+ 50% 16.30ms+ 75% 18.19ms+ 90% 20.30ms+ 99% 24.95ms+ 1535 requests in 801.08ms, 7.93MB read+Requests/sec: 1916.18+Transfer/sec: 9.90MB```-The results for llama.cpp were:+The results for llama.cpp in CPU mode were:```+ 32 threads and 32 connectionsThread Stats Avg Stdev Max +/- Stdev- Latency 262.34ms 75.10ms 322.91ms 82.80%- Req/Sec 3.73 1.88 10.00 93.55%+ Latency 41.16ms 20.70ms 107.46ms 59.20%+ Req/Sec 24.14 8.80 40.00 73.44%Latency Distribution- 50% 281.00ms- 75% 321.60ms- 90% 322.32ms- 99% 322.91ms- 93 requests in 901.53ms, 748.74KB read-Requests/sec: 103.16-Transfer/sec: 830.53KB+ 50% 51.99ms+ 75% 56.98ms+ 90% 60.19ms+ 99% 87.11ms+ 2163 requests in 2.80s, 16.96MB read+Requests/sec: 771.50+Transfer/sec: 6.05MB```-It's possible to make the llama.cpp server go faster, by passing the-`-np 10` flag. This will preallocate ten server slots which allow-requests to be handled in parallel. However doing this boosted the-performance to no more than 200 requests per second. Specifying higher-values, e.g. `-np 50` causes the llama.cpp to crash.+GPU mode was also attempted with a pair of RTX 4090s. However it was+only able to do 450 requests per second. CPU goes faster here probably+because we're using a model that's small enough to fit entirely inside+Threadripper's L3 cache.For less computationally intensive endpoints, such as `/tokenize`,llamafiler on Threadripper PRO 7995WX is able to serve 3.3 million@@ -95,19 +95,24 @@ LLaMAfiler uses `pthread_cancel()` to asynchronously cancel requestswhen appropriate. The server is designed this way so that high-priorityor latency-sensitive requests can preempt batch background jobs.-For example, we intend to implement an HTTP header that will allow a-client to elect the priority of its request. For low-priority requests,+Clients can voluntarily send an `X-Priority: batch` HTTP header that'll+make a request first in line for preemption. For low-priority requests,if a higher priority job happens to come in, the low-priority job can becanceled in less than a millisecond, even if it's inside a deep,long-running matrix multiplication operation. This will not leak memory.It ensures that resources are freed up immediately for the other job.-Cancelation is also useful for DDoS mitigation. The `-w N` flag allows-you to specify a fixed number of HTTP workers. By default, this will be-set to the number of CPU cores multiplied by two. HTTP clients are-permitted to idle (via the HTTP keep-alive mechanism) as long as they-wish. However, if the number of available workers runs out when a new-request comes in, the oldest worker will be canceled to make room.+Clients can also be deprioritized involuntarily, if they create more+connections or send more requests than the token bucket burst limit.+This provides a last line of defense against DDOS. Even without tokens,+clients can still do whatever they want, because no action is taken by+the server until resource pressure happens. The `-w N` flag allows you+to specify a fixed number of HTTP workers. By default, this will be set+to the number of CPU cores multiplied by two. HTTP clients are permitted+to idle (via the HTTP keep-alive mechanism) or spam requests as long and+as much as they wish. However, if the number of available workers runs+out when a new request comes in, then the most recently deprioritized+worker will be canceled to make room.When a cancelation happens, HTTP connections in the idle state willsimply be closed. If an HTTP connection is in the middle of serving a@@ -118,11 +123,12 @@ multiple servers, then failover strategies can also be used.## Crash Proofing-LLaMAfiler is designed to be impossible to crash. Let's say there's a-bug in llama.cpp or LLaMAfiler that causes a SIGSEGV to happen while-processing a request. In that case, the worker thread will be canceled-and a backtrace will be logged. A fresh new worker thread will then be-spawned to take its place.+One of the issues with the upstream llama.cpp server is that it's very+easy to make it crash. LLaMAfiler is designed to be impossible to crash.+Let's say there's a bug in llama.cpp or LLaMAfiler that causes a SIGSEGV+to happen while processing a request. In that case, the worker thread+will be canceled and a backtrace will be logged. A fresh new worker+thread will then be spawned to take its place.Each thread is given a stack size of 81KB, which includes the guardpage. Stack overflows are reported and recoverable with a backtrace,diff --git a/llamafile/server/main.cpp b/llamafile/server/main.cppindex 9f7055de..8293058b 100644--- a/llamafile/server/main.cpp+++ b/llamafile/server/main.cpp@@ -28,6 +28,7 @@#include "server.h"#include "signals.h"#include "time.h"+#include "tokenbucket.h"Server* g_server;llama_model* g_model;@@ -46,6 +47,7 @@ main(int argc, char* argv[])LoadZipArgs(&argc, &argv);llamafile_get_flags(argc, argv);time_init();+ tokenbucket_init();// we must disable the llama.cpp logger// otherwise pthread_cancel() will cause deadlocks@@ -78,6 +80,16 @@ main(int argc, char* argv[])for (int i = 0; i < FLAG_workers; ++i)npassert(!g_server->spawn());+ // install security+ if (!FLAG_unsecure) {+ if (pledge(0, 0)) {+ SLOG("warning: this OS doesn't support pledge() security\n");+ } else if (pledge("stdio anet", 0)) {+ perror("pledge");+ exit(1);+ }+ }+// run serversignals_init();llama_backend_init();@@ -91,6 +103,7 @@ main(int argc, char* argv[])g_server->close();delete g_server;llama_free_model(g_model);+ tokenbucket_destroy();time_destroy();SLOG("exit");diff --git a/llamafile/server/server.cpp b/llamafile/server/server.cppindex d3cf36ac..b4d0fa57 100644--- a/llamafile/server/server.cpp+++ b/llamafile/server/server.cpp@@ -114,7 +114,7 @@ Server::spawn()}int-Server::accept()+Server::accept(unsigned* out_ip){// accept connectionsockaddr_in clientaddr;@@ -148,6 +148,8 @@ Server::accept()if (FLAG_verbose >= 2)SLOG("accept");+ if (out_ip)+ *out_ip = ip;return clifd;}diff --git a/llamafile/server/server.h b/llamafile/server/server.hindex 967909fd..e9b09ff8 100644--- a/llamafile/server/server.h+++ b/llamafile/server/server.h@@ -25,7 +25,7 @@ struct ServerServer(int);~Server();- int accept();+ int accept(unsigned*);errno_t spawn();void terminate();void shutdown();diff --git a/llamafile/server/time.cpp b/llamafile/server/time.cppindex 7dc98cf8..2ac0c9d8 100644--- a/llamafile/server/time.cpp+++ b/llamafile/server/time.cpp@@ -146,18 +146,13 @@ voidtime_init(){update_time();- pthread_attr_t attr;- pthread_attr_init(&attr);- if (pthread_create(&g_time_thread, &attr, time_worker, 0))+ if (pthread_create(&g_time_thread, 0, time_worker, 0))__builtin_trap();- pthread_attr_destroy(&attr);}voidtime_destroy(){- if (!g_time_thread)- return;pthread_cancel(g_time_thread);if (pthread_join(g_time_thread, 0))__builtin_trap();diff --git a/llamafile/server/tokenbucket.cpp b/llamafile/server/tokenbucket.cppnew file mode 100644index 00000000..1e5d17bf--- /dev/null+++ b/llamafile/server/tokenbucket.cpp@@ -0,0 +1,125 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "tokenbucket.h"++#include <assert.h>+#include <pthread.h>+#include <signal.h>+#include <stdatomic.h>++#include "llamafile/llamafile.h"+#include "llamafile/server/log.h"++union TokenBucket+{+ atomic_uint_fast64_t* w;+ atomic_schar* b;+};++static size_t g_words;+static pthread_t g_thread;+static TokenBucket g_tokens;++static void+replenish_tokens(atomic_uint_fast64_t* w, size_t n)+{+ for (size_t i = 0; i < n; ++i) {+ uint64_t a = atomic_load_explicit(w + i, memory_order_relaxed);+ if (a == 0x7f7f7f7f7f7f7f7f)+ continue;+ uint64_t b = 0x8080808080808080;+ uint64_t c = 0x7f7f7f7f7f7f7f7f ^ a;+ uint64_t d = ((((c >> 1 | b) - c) & b) ^ b) >> 7;+ atomic_fetch_add_explicit(w + i, d, memory_order_relaxed);+ }+}++static int+acquire_token(atomic_schar* b, uint32_t x, int c)+{+ uint32_t i = x >> (32 - c);+ int t = atomic_load_explicit(b + i, memory_order_relaxed);+ if (t > 0)+ t = atomic_fetch_add_explicit(b + i, -1, memory_order_relaxed);+ return -t + 127;+}++void+tokenbucket_replenish()+{+ replenish_tokens(g_tokens.w, g_words);+}++static void+tokenbucket_replenisher()+{+ timespec rate = timespec_frommicros(1e6 / FLAG_token_rate);+ timespec next = timespec_add(timespec_real(), rate);+ for (;;) {+ while (timespec_cmp(timespec_real(), next) > 0) {+ tokenbucket_replenish();+ next = timespec_add(next, rate);+ }+ clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &next, 0);+ }+}++static void*+tokenbucket_thread(void* arg)+{+ sigset_t ss;+ sigemptyset(&ss);+ sigaddset(&ss, SIGHUP);+ sigaddset(&ss, SIGINT);+ sigaddset(&ss, SIGQUIT);+ sigaddset(&ss, SIGTERM);+ sigaddset(&ss, SIGUSR1);+ sigaddset(&ss, SIGALRM);+ pthread_sigmask(SIG_SETMASK, &ss, 0);+ set_thread_name("tokenbucket");+ tokenbucket_replenisher();+ return nullptr;+}++void+tokenbucket_init()+{+ size_t bytes = 1;+ bytes <<= FLAG_token_cidr;+ g_words = bytes / 8;+ if (!(g_tokens.b = (atomic_schar*)malloc(bytes)))+ __builtin_trap();+ memset(g_tokens.b, 127, bytes);+ if (pthread_create(&g_thread, 0, tokenbucket_thread, 0))+ __builtin_trap();+}++void+tokenbucket_destroy()+{+ pthread_cancel(g_thread);+ if (pthread_join(g_thread, 0))+ __builtin_trap();+ free(g_tokens.b);+}++int+tokenbucket_acquire(unsigned ip)+{+ return acquire_token(g_tokens.b, ip, FLAG_token_cidr);+}diff --git a/llamafile/server/tokenbucket.h b/llamafile/server/tokenbucket.hnew file mode 100644index 00000000..62fb290d--- /dev/null+++ b/llamafile/server/tokenbucket.h@@ -0,0 +1,30 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once++void+tokenbucket_init();++void+tokenbucket_destroy();++int+tokenbucket_acquire(unsigned);++void+tokenbucket_replenish();diff --git a/llamafile/server/tokenbucket_test.cpp b/llamafile/server/tokenbucket_test.cppnew file mode 100644index 00000000..789e4b87--- /dev/null+++ b/llamafile/server/tokenbucket_test.cpp@@ -0,0 +1,64 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "tokenbucket.h"++#include <cosmo.h>+#include <limits.h>++#include "llamafile/llamafile.h"++int+main(int argc, char* argv[])+{+ FLAG_token_rate = .0001;+ FLAG_token_cidr = 24;+ tokenbucket_init();++ if (tokenbucket_acquire(0x7f000001) != 0)+ return 1;+ if (tokenbucket_acquire(0x7f000002) != 1)+ return 1;+ if (tokenbucket_acquire(0x7f000003) != 2)+ return 1;+ if (tokenbucket_acquire(0x7f000100) != 0)+ return 1;+ if (tokenbucket_acquire(0x7f000101) != 1)+ return 1;++ tokenbucket_replenish();+ tokenbucket_replenish();++ if (tokenbucket_acquire(0x7f000003) != 1)+ return 1;+ if (tokenbucket_acquire(0x7f000003) != 2)+ return 1;+ if (tokenbucket_acquire(0x7f000101) != 0)+ return 1;++ tokenbucket_replenish();+ tokenbucket_replenish();+ tokenbucket_replenish();++ if (tokenbucket_acquire(0x7f000003) != 0)+ return 1;+ if (tokenbucket_acquire(0x7f000101) != 0)+ return 1;++ tokenbucket_destroy();+ CheckForMemoryLeaks();+}diff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cppindex 2750225f..9152bb5e 100644--- a/llamafile/server/worker.cpp+++ b/llamafile/server/worker.cpp@@ -23,10 +23,12 @@#include "llamafile/llamafile.h"#include "llamafile/threadlocal.h"+#include "llamafile/trust.h"#include "client.h"#include "log.h"#include "signals.h"+#include "tokenbucket.h"Worker::Worker(Server* server) : server(server){@@ -43,6 +45,11 @@ voidWorker::begin(){npassert(!working);+ client.worker = this;+ client.client_ip_trusted = is_trusted_ip(client.client_ip);+ int tokens = 0;+ if (!client.client_ip_trusted)+ tokens = tokenbucket_acquire(client.client_ip);server->lock();dll_remove(&server->idle_workers, &elem);if (dll_is_empty(server->idle_workers)) {@@ -53,7 +60,11 @@ Worker::begin()}}working = true;- dll_make_first(&server->active_workers, &elem);+ if (tokens > FLAG_token_burst) {+ dll_make_last(&server->active_workers, &elem);+ } else {+ dll_make_first(&server->active_workers, &elem);+ }server->unlock();}@@ -68,6 +79,16 @@ Worker::end()server->unlock();}+void+Worker::deprioritize()+{+ npassert(working);+ server->lock();+ dll_remove(&server->active_workers, &elem);+ dll_make_last(&server->active_workers, &elem);+ server->unlock();+}+voidWorker::retire(){@@ -83,9 +104,9 @@ Worker::retire()}void-Worker::handle(void)+Worker::handle(){- if ((client.fd = server->accept()) == -1) {+ if ((client.fd = server->accept(&client.client_ip)) == -1) {SLOG("accept returned %m");return;}diff --git a/llamafile/server/worker.h b/llamafile/server/worker.hindex 1b65bff4..a76ccb61 100644--- a/llamafile/server/worker.h+++ b/llamafile/server/worker.h@@ -34,8 +34,9 @@ struct WorkerWorker(Server*);void run();void begin();- void handle(void);+ void handle();void end();+ void deprioritize();void retire();void kill();};diff --git a/llamafile/trust.cpp b/llamafile/trust.cppnew file mode 100644index 00000000..7266d076--- /dev/null+++ b/llamafile/trust.cpp@@ -0,0 +1,27 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "trust.h"++ctl::vector<cidr> FLAG_trust;++bool is_trusted_ip(unsigned ip) noexcept {+ for (auto c : FLAG_trust)+ if (c.matches(ip))+ return true;+ return false;+}diff --git a/llamafile/trust.h b/llamafile/trust.hnew file mode 100644index 00000000..7443f40d--- /dev/null+++ b/llamafile/trust.h@@ -0,0 +1,37 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <ctl/string.h>+#include <ctl/vector.h>++struct cidr {+ unsigned ip;+ unsigned bits;++ constexpr bool matches(unsigned x) const noexcept {+ unsigned mask = -1u << (32 - bits);+ return (x & mask) == (ip & mask);+ }+};++extern ctl::vector<cidr> FLAG_trust;++bool is_trusted_ip(unsigned) noexcept;+bool is_loopback_ip(unsigned) noexcept;+long parse_ip(const ctl::string_view &) noexcept;+bool parse_cidr(const ctl::string_view &, cidr *) noexcept;