Skip to content

Distribute and run LLMs with a single file.

Add seccomp, tokenbucket, and batch prioritization

Justine Tunney99dd1c01084e
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 86dae40a..af936585 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -17,6 +17,8 @@ LLAMAFILE_OBJS := \
$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \
$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)
+$(LLAMAFILE_OBJS): private CCFLAGS += -g
+
# this executable defines its own malloc(), free(), etc.
# therefore we want to avoid it going inside the .a file
LLAMAFILE_OBJS := $(filter-out o/$(MODE)/llamafile/zipalign.o,$(LLAMAFILE_OBJS))
@@ -52,6 +54,7 @@ o/$(MODE)/llamafile: \
o/$(MODE)/llamafile/addnl \
o/$(MODE)/llamafile/pool_test.runs \
o/$(MODE)/llamafile/pool_cancel_test.runs \
+ o/$(MODE)/llamafile/parse_cidr_test.runs \
################################################################################
# microarchitectures
@@ -143,6 +146,11 @@ o/$(MODE)/llamafile/tinyblas_cpu_sgemm_arm82.o: \
################################################################################
# testing
+o/$(MODE)/llamafile/parse_cidr_test: \
+ o/$(MODE)/llamafile/parse_cidr_test.o \
+ o/$(MODE)/llamafile/parse_cidr.o \
+ o/$(MODE)/llamafile/parse_ip.o \
+
o/$(MODE)/llamafile/pool_test: \
o/$(MODE)/llamafile/pool_test.o \
o/$(MODE)/llamafile/crash.o \
diff --git a/llamafile/flags.cpp b/llamafile/flags.cpp
index 761ccf52..d2f0a703 100644
--- a/llamafile/flags.cpp
+++ b/llamafile/flags.cpp
@@ -17,11 +17,15 @@
#include "debug.h"
#include "llamafile.h"
+#include "trust.h"
#include <errno.h>
#include <fcntl.h>
+#include <limits.h>
+#include <math.h>
#include <stdio.h>
#include <stdlib.h>
+#include <time.h>
#include <unistd.h>
#include "llama.cpp/llama.h"
@@ -38,9 +42,11 @@ bool FLAG_tinyblas = false;
bool FLAG_trace = false;
bool FLAG_unsecure = false;
const char *FLAG_file = nullptr;
+const char *FLAG_ip_header = nullptr;
const char *FLAG_listen = "0.0.0.0:8080";
const char *FLAG_model = nullptr;
const char *FLAG_prompt = nullptr;
+double FLAG_token_rate = 1;
float FLAG_temp = 0.8;
int FLAG_batch = 2048;
int FLAG_ctx = 512;
@@ -54,6 +60,8 @@ int FLAG_n_gpu_layers = -1;
int FLAG_seed = LLAMA_DEFAULT_SEED;
int FLAG_split_mode = LLAMA_SPLIT_MODE_LAYER;
int FLAG_threads;
+int FLAG_token_burst = 100;
+int FLAG_token_cidr = 24;
int FLAG_ubatch = 512;
int FLAG_verbose = 0;
int FLAG_warmup = true;
@@ -136,6 +144,61 @@ void llamafile_get_flags(int argc, char **argv) {
continue;
}
+ if (!strcmp(flag, "--ip-header")) {
+ if (i == argc)
+ missing("--ip-header");
+ cidr ip;
+ FLAG_ip_header = argv[i++];
+ continue;
+ }
+
+ if (!strcmp(flag, "--trust")) {
+ if (i == argc)
+ missing("--trust");
+ cidr ip;
+ if (!parse_cidr(argv[i++], &ip))
+ error("invalid --trust CIDR (expect like 192.168.0.0/24)");
+ FLAG_trust.push_back(ip);
+ continue;
+ }
+
+ if (!strcmp(flag, "--token-rate")) {
+ if (i == argc)
+ missing("--token-rate");
+ double rate = atof(argv[i++]);
+ if (!rate)
+ error("--token-rate can't be zero");
+ double micros = 1e6 / rate;
+ if (isnan(micros) || isinf(micros))
+ error("--token-rate invalid");
+ if (micros < 10)
+ error("--token-rate too frequent (can't be above 100000 a.k.a. 1 per 10µs)");
+ if (micros > 60 * 60 * 1e6)
+ error("--token-rate too infrequent (can't be below 1/(60*60) a.k.a. 1 per hour)");
+ FLAG_token_rate = rate;
+ continue;
+ }
+
+ if (!strcmp(flag, "--token-burst")) {
+ if (i == argc)
+ missing("--token-burst");
+ int burst = atoi(argv[i++]);
+ if (!(1 <= burst && burst <= 127))
+ error("--token-burst must be 1..127");
+ FLAG_token_burst = burst;
+ continue;
+ }
+
+ if (!strcmp(flag, "--token-cidr")) {
+ if (i == argc)
+ missing("--token-cidr");
+ int cidr = atoi(argv[i++]);
+ if (!(3 <= cidr && cidr <= 32))
+ error("--token-cidr must be 3..32");
+ FLAG_token_cidr = cidr;
+ continue;
+ }
+
//////////////////////////////////////////////////////////////////////
// http server flags
diff --git a/llamafile/is_loopback_ip.cpp b/llamafile/is_loopback_ip.cpp
new file mode 100644
index 00000000..314e9d06
--- /dev/null
+++ b/llamafile/is_loopback_ip.cpp
@@ -0,0 +1,22 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "trust.h"
+
+bool is_loopback_ip(unsigned x) noexcept {
+ return (x >> 24) == 127; /* 127.0.0.0/8 */
+}
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 15b0bc20..bc1cf090 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -19,9 +19,11 @@ extern bool FLAG_trace;
extern bool FLAG_trap;
extern bool FLAG_unsecure;
extern const char *FLAG_file;
+extern const char *FLAG_ip_header;
extern const char *FLAG_listen;
extern const char *FLAG_model;
extern const char *FLAG_prompt;
+extern double FLAG_token_rate;
extern float FLAG_temp;
extern int FLAG_batch;
extern int FLAG_ctx;
@@ -36,6 +38,8 @@ extern int FLAG_n_gpu_layers;
extern int FLAG_seed;
extern int FLAG_split_mode;
extern int FLAG_threads;
+extern int FLAG_token_burst;
+extern int FLAG_token_cidr;
extern int FLAG_ubatch;
extern int FLAG_verbose;
extern int FLAG_warmup;
diff --git a/llamafile/parse_cidr.cpp b/llamafile/parse_cidr.cpp
new file mode 100644
index 00000000..f2b08bf6
--- /dev/null
+++ b/llamafile/parse_cidr.cpp
@@ -0,0 +1,54 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "trust.h"
+
+#include <ctype.h>
+#include <string.h>
+
+bool parse_cidr(const ctl::string_view &str, cidr *out_cidr) noexcept {
+ long ip;
+ int bits;
+ size_t i, n;
+ const char *s, *p;
+ s = str.data();
+ n = str.size();
+ if ((p = (const char *)memchr(s, '/', n))) {
+ if ((ip = parse_ip(ctl::string_view(s, (i = p - s)))) == -1)
+ return false;
+ bits = 0;
+ for (++i; i < n; ++i) {
+ if (!isdigit(s[i]))
+ return false;
+ bits *= 10;
+ bits += s[i] - '0';
+ if (bits > 32)
+ return false;
+ }
+ if (bits <= 0)
+ return false;
+ } else {
+ if ((ip = parse_ip(str)) == -1)
+ return false;
+ bits = 32;
+ }
+ if (out_cidr) {
+ out_cidr->ip = ip;
+ out_cidr->bits = bits;
+ }
+ return true;
+}
diff --git a/llamafile/parse_cidr_test.cpp b/llamafile/parse_cidr_test.cpp
new file mode 100644
index 00000000..a7a5bd00
--- /dev/null
+++ b/llamafile/parse_cidr_test.cpp
@@ -0,0 +1,66 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "trust.h"
+
+int main(int argc, char *argv[]) {
+ cidr in;
+
+ if (!parse_cidr("10.10.10.1/24", &in))
+ return 1;
+ if (in.ip != 0x0a0a0a01)
+ return 2;
+ if (in.bits != 24)
+ return 3;
+ if (in.matches(parse_ip("10.10.11.1")))
+ return 3;
+ if (!in.matches(parse_ip("10.10.10.1")))
+ return 3;
+ if (!in.matches(parse_ip(" 10.10.10.255 ")))
+ return 3;
+
+ if (!parse_cidr("168430081/1", &in))
+ return 4;
+ if (in.ip != 0x0a0a0a01)
+ return 5;
+ if (in.bits != 1)
+ return 6;
+
+ if (parse_cidr("", 0))
+ return 77;
+ if (parse_cidr(" ", 0))
+ return 78;
+ if (parse_cidr("10.10.10.1/", 0))
+ return 7;
+ if (parse_cidr("10.10.10.1/a", 0))
+ return 8;
+ if (parse_cidr("10.10.10.1/33", 0))
+ return 9;
+ if (parse_cidr("10.10.10.a", 0))
+ return 10;
+
+ if (!parse_cidr("10.10.10.255", &in))
+ return 11;
+ if (in.ip != 0x0a0a0aff)
+ return 12;
+ if (in.bits != 32)
+ return 13;
+ if (in.matches(parse_ip("10.10.10.254")))
+ return 14;
+ if (!in.matches(parse_ip("10.10.10.255")))
+ return 15;
+}
diff --git a/llamafile/parse_ip.cpp b/llamafile/parse_ip.cpp
new file mode 100644
index 00000000..da49db1e
--- /dev/null
+++ b/llamafile/parse_ip.cpp
@@ -0,0 +1,71 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "trust.h"
+
+#include <ctype.h>
+#include <stdckdint.h>
+#include <string.h>
+
+long parse_ip(const ctl::string_view &str) noexcept {
+ int c, j;
+ size_t i, n;
+ unsigned b, x;
+ const char *s;
+ bool dotted = false;
+ s = str.data();
+ n = str.size();
+
+ // trim leading spaces
+ while (n && isspace(*s)) {
+ ++s;
+ --n;
+ }
+
+ // trim trailing spaces
+ while (n && isspace(s[n - 1]))
+ --n;
+
+ // forbid empty string
+ if (!n)
+ return -1;
+
+ // parse ipv4 address
+ for (b = x = j = i = 0; i < n; ++i) {
+ c = s[i] & 255;
+ if (isdigit(c)) {
+ if (ckd_mul(&b, b, 10) || //
+ ckd_add(&b, b, c - '0') || //
+ (b > 255 && dotted)) {
+ return -1;
+ }
+ } else if (c == '.') {
+ if (b > 255)
+ return -1;
+ dotted = true;
+ x <<= 8;
+ x |= b;
+ b = 0;
+ ++j;
+ } else {
+ return -1;
+ }
+ }
+ x <<= 8;
+ x |= b;
+ return x;
+}
diff --git a/llamafile/server/BUILD.mk b/llamafile/server/BUILD.mk
index c763de4f..37276db3 100644
--- a/llamafile/server/BUILD.mk
+++ b/llamafile/server/BUILD.mk
@@ -9,6 +9,8 @@ LLAMAFILE_SERVER_INCS = $(filter %.inc,$(LLAMAFILE_SERVER_FILES))
LLAMAFILE_SERVER_SRCS = $(filter %.cpp,$(LLAMAFILE_SERVER_FILES))
LLAMAFILE_SERVER_OBJS = $(LLAMAFILE_SERVER_SRCS:%.cpp=o/$(MODE)/%.o)
+$(LLAMAFILE_SERVER_OBJS): private CCFLAGS += -g
+
o/$(MODE)/llamafile/server/server.a: \
$(filter-out %_test.o,$(LLAMAFILE_SERVER_OBJS))
@@ -33,6 +35,12 @@ o/$(MODE)/llamafile/server/json_test: \
o/$(MODE)/llamafile/server/hextoint.o \
o/$(MODE)/double-conversion/double-conversion.a \
+o/$(MODE)/llamafile/server/tokenbucket_test: \
+ o/$(MODE)/llamafile/server/tokenbucket_test.o \
+ o/$(MODE)/llamafile/server/tokenbucket.o \
+ o/$(MODE)/llamafile/server/log.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
+
.PHONY: o/$(MODE)/llamafile/server
o/$(MODE)/llamafile/server: \
o/$(MODE)/llamafile/server/main \
diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cpp
index 546dba45..a28691ca 100644
--- a/llamafile/server/client.cpp
+++ b/llamafile/server/client.cpp
@@ -28,10 +28,13 @@
#include "llama.cpp/llama.h"
#include "llamafile/llamafile.h"
#include "llamafile/threadlocal.h"
+#include "llamafile/trust.h"
#include "llamafile/version.h"
#include "log.h"
#include "time.h"
+#include "tokenbucket.h"
+#include "worker.h"
#define STANDARD_RESPONSE_HEADERS \
"Server: llamafile/" LLAMAFILE_VERSION_STRING "\r\n" \
@@ -188,6 +191,36 @@ Client::transport()
return send_error(417);
}
+ effective_ip = client_ip;
+ effective_ip_trusted = client_ip_trusted;
+ if (FLAG_ip_header) {
+ if (is_loopback_ip(client_ip) || client_ip_trusted) {
+ ctl::string_view ip_header = get_header(FLAG_ip_header);
+ if (!ip_header.empty()) {
+ long ip;
+ if ((ip = parse_ip(ip_header)) == -1) {
+ effective_ip = ip;
+ effective_ip_trusted = is_trusted_ip(ip);
+ } else {
+ SLOG("client's --ip-header wasn't a single ipv4 address");
+ effective_ip_trusted = false;
+ }
+ }
+ } else {
+ SLOG("received direct connect from untrusted ip");
+ effective_ip_trusted = false;
+ }
+ }
+
+ if (get_header("X-Priority") == "batch") {
+ worker->deprioritize();
+ } else if (!effective_ip_trusted) {
+ if (tokenbucket_acquire(client_ip) > FLAG_token_burst) {
+ SLOG("deprioritizing");
+ worker->deprioritize();
+ }
+ }
+
if (HasHeader(kHttpTransferEncoding))
if (!HeaderEqualCase(kHttpTransferEncoding, "identity")) {
close_connection = true;
@@ -354,6 +387,28 @@ Client::has_at_most_this_element(int h, const string_view s)
return true;
}
+ctl::string_view
+Client::get_header(const ctl::string_view& key)
+{
+ int h;
+ size_t i, keylen;
+ if ((h = GetHttpHeader(key.data(), key.size())) != -1) {
+ if (msg.headers[h].a)
+ return ctl::string_view(ibuf.p + msg.headers[h].a,
+ msg.headers[h].b - msg.headers[h].a);
+ } else {
+ for (i = 0; i < msg.xheaders.n; ++i)
+ if (SlicesEqualCase(key.data(),
+ key.size(),
+ ibuf.p + msg.xheaders.p[i].k.a,
+ msg.xheaders.p[i].k.b - msg.xheaders.p[i].k.a))
+ return ctl::string_view(ibuf.p + msg.xheaders.p[i].v.a,
+ msg.xheaders.p[i].v.b -
+ msg.xheaders.p[i].v.a);
+ }
+ return ctl::string_view();
+}
+
bool
Client::read_payload()
{
diff --git a/llamafile/server/client.h b/llamafile/server/client.h
index 0b16f642..407dc349 100644
--- a/llamafile/server/client.h
+++ b/llamafile/server/client.h
@@ -36,6 +36,7 @@
#define HeaderEqualCase(H, S) \
SlicesEqualCase(S, strlen(S), HeaderData(H), HeaderLength(H))
+struct Worker;
struct TokenizeParams;
struct EmbeddingParams;
struct CompletionParams;
@@ -50,9 +51,14 @@ struct Cleanup
struct Client
{
int fd = -1;
+ unsigned client_ip = 0;
+ unsigned effective_ip = 0;
+ bool client_ip_trusted = false;
+ bool effective_ip_trusted = false;
bool close_connection = false;
bool should_send_error_if_canceled;
size_t unread = 0;
+ Worker* worker;
timespec message_started;
HttpMessage msg;
Url url = {};
@@ -84,6 +90,7 @@ struct Client
bool send2(const ctl::string_view, const ctl::string_view) __wur;
char* append_header(const ctl::string_view, const ctl::string_view);
bool has_at_most_this_element(int, const ctl::string_view);
+ ctl::string_view get_header(const ctl::string_view&);
ctl::string_view path();
ctl::optional<ctl::string_view> param(ctl::string_view);
diff --git a/llamafile/server/doc/technical_details.md b/llamafile/server/doc/technical_details.md
index c8ca1a49..f58f9450 100644
--- a/llamafile/server/doc/technical_details.md
+++ b/llamafile/server/doc/technical_details.md
@@ -9,8 +9,8 @@ technical aspects of LLaMAfiler.
This server can serve embeddings an order of a magnitude faster than the
llama.cpp example server. We measured this on Threadripper PRO 7995WX w/
all-MiniLM-L6-v2 Q6\_K. Using a prompt with 50 tokens this server served
-930 `/embedding` requests per second, whereas llama.cpp/build/bin/server
-built with glibc served 100 per second.
+1,916 `/embedding` requests per second, whereas
+llama.cpp/build/bin/server built with glibc served 771 per second.
The following wrk script was used:
@@ -23,10 +23,10 @@ wrk.headers["Content-Type"] = "application/json"
With the following commands:
```
-llamafiler -m /weights/all-MiniLM-L6-v2.Q6_K.gguf
+llamafiler -m /weights/all-MiniLM-L6-v2.Q6_K.gguf --trust 127.0.0.1/32
wrk --latency -t 32 -c 32 -s embedding.lua 'http://127.0.0.1:8080/embedding'
-~/llama.cpp/build/bin/server -m /weights/all-MiniLM-L6-v2.Q6_K.gguf --embeddings
+~/llama.cpp/llama-server -m /weights/all-MiniLM-L6-v2.Q6_K.gguf --embeddings
wrk --latency -t 32 -c 32 -s embedding.lua 'http://127.0.0.1:8080/embedding'
```
@@ -34,39 +34,39 @@ The results for llamafiler were:
```
Thread Stats Avg Stdev Max +/- Stdev
- Latency 33.89ms 7.48ms 61.55ms 66.90%
- Req/Sec 29.26 6.52 50.00 59.74%
+ Latency 16.50ms 3.01ms 31.95ms 71.14%
+ Req/Sec 59.95 7.17 80.00 52.34%
Latency Distribution
- 50% 33.31ms
- 75% 38.76ms
- 90% 43.76ms
- 99% 53.31ms
- 1592 requests in 1.70s, 8.26MB read
-Requests/sec: 934.80
-Transfer/sec: 4.85MB
+ 50% 16.30ms
+ 75% 18.19ms
+ 90% 20.30ms
+ 99% 24.95ms
+ 1535 requests in 801.08ms, 7.93MB read
+Requests/sec: 1916.18
+Transfer/sec: 9.90MB
```
-The results for llama.cpp were:
+The results for llama.cpp in CPU mode were:
```
+ 32 threads and 32 connections
Thread Stats Avg Stdev Max +/- Stdev
- Latency 262.34ms 75.10ms 322.91ms 82.80%
- Req/Sec 3.73 1.88 10.00 93.55%
+ Latency 41.16ms 20.70ms 107.46ms 59.20%
+ Req/Sec 24.14 8.80 40.00 73.44%
Latency Distribution
- 50% 281.00ms
- 75% 321.60ms
- 90% 322.32ms
- 99% 322.91ms
- 93 requests in 901.53ms, 748.74KB read
-Requests/sec: 103.16
-Transfer/sec: 830.53KB
+ 50% 51.99ms
+ 75% 56.98ms
+ 90% 60.19ms
+ 99% 87.11ms
+ 2163 requests in 2.80s, 16.96MB read
+Requests/sec: 771.50
+Transfer/sec: 6.05MB
```
-It's possible to make the llama.cpp server go faster, by passing the
-`-np 10` flag. This will preallocate ten server slots which allow
-requests to be handled in parallel. However doing this boosted the
-performance to no more than 200 requests per second. Specifying higher
-values, e.g. `-np 50` causes the llama.cpp to crash.
+GPU mode was also attempted with a pair of RTX 4090s. However it was
+only able to do 450 requests per second. CPU goes faster here probably
+because we're using a model that's small enough to fit entirely inside
+Threadripper's L3 cache.
For less computationally intensive endpoints, such as `/tokenize`,
llamafiler on Threadripper PRO 7995WX is able to serve 3.3 million
@@ -95,19 +95,24 @@ LLaMAfiler uses `pthread_cancel()` to asynchronously cancel requests
when appropriate. The server is designed this way so that high-priority
or latency-sensitive requests can preempt batch background jobs.
-For example, we intend to implement an HTTP header that will allow a
-client to elect the priority of its request. For low-priority requests,
+Clients can voluntarily send an `X-Priority: batch` HTTP header that'll
+make a request first in line for preemption. For low-priority requests,
if a higher priority job happens to come in, the low-priority job can be
canceled in less than a millisecond, even if it's inside a deep,
long-running matrix multiplication operation. This will not leak memory.
It ensures that resources are freed up immediately for the other job.
-Cancelation is also useful for DDoS mitigation. The `-w N` flag allows
-you to specify a fixed number of HTTP workers. By default, this will be
-set to the number of CPU cores multiplied by two. HTTP clients are
-permitted to idle (via the HTTP keep-alive mechanism) as long as they
-wish. However, if the number of available workers runs out when a new
-request comes in, the oldest worker will be canceled to make room.
+Clients can also be deprioritized involuntarily, if they create more
+connections or send more requests than the token bucket burst limit.
+This provides a last line of defense against DDOS. Even without tokens,
+clients can still do whatever they want, because no action is taken by
+the server until resource pressure happens. The `-w N` flag allows you
+to specify a fixed number of HTTP workers. By default, this will be set
+to the number of CPU cores multiplied by two. HTTP clients are permitted
+to idle (via the HTTP keep-alive mechanism) or spam requests as long and
+as much as they wish. However, if the number of available workers runs
+out when a new request comes in, then the most recently deprioritized
+worker will be canceled to make room.
When a cancelation happens, HTTP connections in the idle state will
simply be closed. If an HTTP connection is in the middle of serving a
@@ -118,11 +123,12 @@ multiple servers, then failover strategies can also be used.
## Crash Proofing
-LLaMAfiler is designed to be impossible to crash. Let's say there's a
-bug in llama.cpp or LLaMAfiler that causes a SIGSEGV to happen while
-processing a request. In that case, the worker thread will be canceled
-and a backtrace will be logged. A fresh new worker thread will then be
-spawned to take its place.
+One of the issues with the upstream llama.cpp server is that it's very
+easy to make it crash. LLaMAfiler is designed to be impossible to crash.
+Let's say there's a bug in llama.cpp or LLaMAfiler that causes a SIGSEGV
+to happen while processing a request. In that case, the worker thread
+will be canceled and a backtrace will be logged. A fresh new worker
+thread will then be spawned to take its place.
Each thread is given a stack size of 81KB, which includes the guard
page. Stack overflows are reported and recoverable with a backtrace,
diff --git a/llamafile/server/main.cpp b/llamafile/server/main.cpp
index 9f7055de..8293058b 100644
--- a/llamafile/server/main.cpp
+++ b/llamafile/server/main.cpp
@@ -28,6 +28,7 @@
#include "server.h"
#include "signals.h"
#include "time.h"
+#include "tokenbucket.h"
Server* g_server;
llama_model* g_model;
@@ -46,6 +47,7 @@ main(int argc, char* argv[])
LoadZipArgs(&argc, &argv);
llamafile_get_flags(argc, argv);
time_init();
+ tokenbucket_init();
// we must disable the llama.cpp logger
// otherwise pthread_cancel() will cause deadlocks
@@ -78,6 +80,16 @@ main(int argc, char* argv[])
for (int i = 0; i < FLAG_workers; ++i)
npassert(!g_server->spawn());
+ // install security
+ if (!FLAG_unsecure) {
+ if (pledge(0, 0)) {
+ SLOG("warning: this OS doesn't support pledge() security\n");
+ } else if (pledge("stdio anet", 0)) {
+ perror("pledge");
+ exit(1);
+ }
+ }
+
// run server
signals_init();
llama_backend_init();
@@ -91,6 +103,7 @@ main(int argc, char* argv[])
g_server->close();
delete g_server;
llama_free_model(g_model);
+ tokenbucket_destroy();
time_destroy();
SLOG("exit");
diff --git a/llamafile/server/server.cpp b/llamafile/server/server.cpp
index d3cf36ac..b4d0fa57 100644
--- a/llamafile/server/server.cpp
+++ b/llamafile/server/server.cpp
@@ -114,7 +114,7 @@ Server::spawn()
}
int
-Server::accept()
+Server::accept(unsigned* out_ip)
{
// accept connection
sockaddr_in clientaddr;
@@ -148,6 +148,8 @@ Server::accept()
if (FLAG_verbose >= 2)
SLOG("accept");
+ if (out_ip)
+ *out_ip = ip;
return clifd;
}
diff --git a/llamafile/server/server.h b/llamafile/server/server.h
index 967909fd..e9b09ff8 100644
--- a/llamafile/server/server.h
+++ b/llamafile/server/server.h
@@ -25,7 +25,7 @@ struct Server
Server(int);
~Server();
- int accept();
+ int accept(unsigned*);
errno_t spawn();
void terminate();
void shutdown();
diff --git a/llamafile/server/time.cpp b/llamafile/server/time.cpp
index 7dc98cf8..2ac0c9d8 100644
--- a/llamafile/server/time.cpp
+++ b/llamafile/server/time.cpp
@@ -146,18 +146,13 @@ void
time_init()
{
update_time();
- pthread_attr_t attr;
- pthread_attr_init(&attr);
- if (pthread_create(&g_time_thread, &attr, time_worker, 0))
+ if (pthread_create(&g_time_thread, 0, time_worker, 0))
__builtin_trap();
- pthread_attr_destroy(&attr);
}
void
time_destroy()
{
- if (!g_time_thread)
- return;
pthread_cancel(g_time_thread);
if (pthread_join(g_time_thread, 0))
__builtin_trap();
diff --git a/llamafile/server/tokenbucket.cpp b/llamafile/server/tokenbucket.cpp
new file mode 100644
index 00000000..1e5d17bf
--- /dev/null
+++ b/llamafile/server/tokenbucket.cpp
@@ -0,0 +1,125 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenbucket.h"
+
+#include <assert.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdatomic.h>
+
+#include "llamafile/llamafile.h"
+#include "llamafile/server/log.h"
+
+union TokenBucket
+{
+ atomic_uint_fast64_t* w;
+ atomic_schar* b;
+};
+
+static size_t g_words;
+static pthread_t g_thread;
+static TokenBucket g_tokens;
+
+static void
+replenish_tokens(atomic_uint_fast64_t* w, size_t n)
+{
+ for (size_t i = 0; i < n; ++i) {
+ uint64_t a = atomic_load_explicit(w + i, memory_order_relaxed);
+ if (a == 0x7f7f7f7f7f7f7f7f)
+ continue;
+ uint64_t b = 0x8080808080808080;
+ uint64_t c = 0x7f7f7f7f7f7f7f7f ^ a;
+ uint64_t d = ((((c >> 1 | b) - c) & b) ^ b) >> 7;
+ atomic_fetch_add_explicit(w + i, d, memory_order_relaxed);
+ }
+}
+
+static int
+acquire_token(atomic_schar* b, uint32_t x, int c)
+{
+ uint32_t i = x >> (32 - c);
+ int t = atomic_load_explicit(b + i, memory_order_relaxed);
+ if (t > 0)
+ t = atomic_fetch_add_explicit(b + i, -1, memory_order_relaxed);
+ return -t + 127;
+}
+
+void
+tokenbucket_replenish()
+{
+ replenish_tokens(g_tokens.w, g_words);
+}
+
+static void
+tokenbucket_replenisher()
+{
+ timespec rate = timespec_frommicros(1e6 / FLAG_token_rate);
+ timespec next = timespec_add(timespec_real(), rate);
+ for (;;) {
+ while (timespec_cmp(timespec_real(), next) > 0) {
+ tokenbucket_replenish();
+ next = timespec_add(next, rate);
+ }
+ clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &next, 0);
+ }
+}
+
+static void*
+tokenbucket_thread(void* arg)
+{
+ sigset_t ss;
+ sigemptyset(&ss);
+ sigaddset(&ss, SIGHUP);
+ sigaddset(&ss, SIGINT);
+ sigaddset(&ss, SIGQUIT);
+ sigaddset(&ss, SIGTERM);
+ sigaddset(&ss, SIGUSR1);
+ sigaddset(&ss, SIGALRM);
+ pthread_sigmask(SIG_SETMASK, &ss, 0);
+ set_thread_name("tokenbucket");
+ tokenbucket_replenisher();
+ return nullptr;
+}
+
+void
+tokenbucket_init()
+{
+ size_t bytes = 1;
+ bytes <<= FLAG_token_cidr;
+ g_words = bytes / 8;
+ if (!(g_tokens.b = (atomic_schar*)malloc(bytes)))
+ __builtin_trap();
+ memset(g_tokens.b, 127, bytes);
+ if (pthread_create(&g_thread, 0, tokenbucket_thread, 0))
+ __builtin_trap();
+}
+
+void
+tokenbucket_destroy()
+{
+ pthread_cancel(g_thread);
+ if (pthread_join(g_thread, 0))
+ __builtin_trap();
+ free(g_tokens.b);
+}
+
+int
+tokenbucket_acquire(unsigned ip)
+{
+ return acquire_token(g_tokens.b, ip, FLAG_token_cidr);
+}
diff --git a/llamafile/server/tokenbucket.h b/llamafile/server/tokenbucket.h
new file mode 100644
index 00000000..62fb290d
--- /dev/null
+++ b/llamafile/server/tokenbucket.h
@@ -0,0 +1,30 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+void
+tokenbucket_init();
+
+void
+tokenbucket_destroy();
+
+int
+tokenbucket_acquire(unsigned);
+
+void
+tokenbucket_replenish();
diff --git a/llamafile/server/tokenbucket_test.cpp b/llamafile/server/tokenbucket_test.cpp
new file mode 100644
index 00000000..789e4b87
--- /dev/null
+++ b/llamafile/server/tokenbucket_test.cpp
@@ -0,0 +1,64 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenbucket.h"
+
+#include <cosmo.h>
+#include <limits.h>
+
+#include "llamafile/llamafile.h"
+
+int
+main(int argc, char* argv[])
+{
+ FLAG_token_rate = .0001;
+ FLAG_token_cidr = 24;
+ tokenbucket_init();
+
+ if (tokenbucket_acquire(0x7f000001) != 0)
+ return 1;
+ if (tokenbucket_acquire(0x7f000002) != 1)
+ return 1;
+ if (tokenbucket_acquire(0x7f000003) != 2)
+ return 1;
+ if (tokenbucket_acquire(0x7f000100) != 0)
+ return 1;
+ if (tokenbucket_acquire(0x7f000101) != 1)
+ return 1;
+
+ tokenbucket_replenish();
+ tokenbucket_replenish();
+
+ if (tokenbucket_acquire(0x7f000003) != 1)
+ return 1;
+ if (tokenbucket_acquire(0x7f000003) != 2)
+ return 1;
+ if (tokenbucket_acquire(0x7f000101) != 0)
+ return 1;
+
+ tokenbucket_replenish();
+ tokenbucket_replenish();
+ tokenbucket_replenish();
+
+ if (tokenbucket_acquire(0x7f000003) != 0)
+ return 1;
+ if (tokenbucket_acquire(0x7f000101) != 0)
+ return 1;
+
+ tokenbucket_destroy();
+ CheckForMemoryLeaks();
+}
diff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cpp
index 2750225f..9152bb5e 100644
--- a/llamafile/server/worker.cpp
+++ b/llamafile/server/worker.cpp
@@ -23,10 +23,12 @@
#include "llamafile/llamafile.h"
#include "llamafile/threadlocal.h"
+#include "llamafile/trust.h"
#include "client.h"
#include "log.h"
#include "signals.h"
+#include "tokenbucket.h"
Worker::Worker(Server* server) : server(server)
{
@@ -43,6 +45,11 @@ void
Worker::begin()
{
npassert(!working);
+ client.worker = this;
+ client.client_ip_trusted = is_trusted_ip(client.client_ip);
+ int tokens = 0;
+ if (!client.client_ip_trusted)
+ tokens = tokenbucket_acquire(client.client_ip);
server->lock();
dll_remove(&server->idle_workers, &elem);
if (dll_is_empty(server->idle_workers)) {
@@ -53,7 +60,11 @@ Worker::begin()
}
}
working = true;
- dll_make_first(&server->active_workers, &elem);
+ if (tokens > FLAG_token_burst) {
+ dll_make_last(&server->active_workers, &elem);
+ } else {
+ dll_make_first(&server->active_workers, &elem);
+ }
server->unlock();
}
@@ -68,6 +79,16 @@ Worker::end()
server->unlock();
}
+void
+Worker::deprioritize()
+{
+ npassert(working);
+ server->lock();
+ dll_remove(&server->active_workers, &elem);
+ dll_make_last(&server->active_workers, &elem);
+ server->unlock();
+}
+
void
Worker::retire()
{
@@ -83,9 +104,9 @@ Worker::retire()
}
void
-Worker::handle(void)
+Worker::handle()
{
- if ((client.fd = server->accept()) == -1) {
+ if ((client.fd = server->accept(&client.client_ip)) == -1) {
SLOG("accept returned %m");
return;
}
diff --git a/llamafile/server/worker.h b/llamafile/server/worker.h
index 1b65bff4..a76ccb61 100644
--- a/llamafile/server/worker.h
+++ b/llamafile/server/worker.h
@@ -34,8 +34,9 @@ struct Worker
Worker(Server*);
void run();
void begin();
- void handle(void);
+ void handle();
void end();
+ void deprioritize();
void retire();
void kill();
};
diff --git a/llamafile/trust.cpp b/llamafile/trust.cpp
new file mode 100644
index 00000000..7266d076
--- /dev/null
+++ b/llamafile/trust.cpp
@@ -0,0 +1,27 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "trust.h"
+
+ctl::vector<cidr> FLAG_trust;
+
+bool is_trusted_ip(unsigned ip) noexcept {
+ for (auto c : FLAG_trust)
+ if (c.matches(ip))
+ return true;
+ return false;
+}
diff --git a/llamafile/trust.h b/llamafile/trust.h
new file mode 100644
index 00000000..7443f40d
--- /dev/null
+++ b/llamafile/trust.h
@@ -0,0 +1,37 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <ctl/string.h>
+#include <ctl/vector.h>
+
+struct cidr {
+ unsigned ip;
+ unsigned bits;
+
+ constexpr bool matches(unsigned x) const noexcept {
+ unsigned mask = -1u << (32 - bits);
+ return (x & mask) == (ip & mask);
+ }
+};
+
+extern ctl::vector<cidr> FLAG_trust;
+
+bool is_trusted_ip(unsigned) noexcept;
+bool is_loopback_ip(unsigned) noexcept;
+long parse_ip(const ctl::string_view &) noexcept;
+bool parse_cidr(const ctl::string_view &, cidr *) noexcept;