Skip to content

Distribute and run LLMs with a single file.

Refactor new server and get leak checker working

Justine Tunney46dda4f8cc80
diff --git a/llamafile/flags.cpp b/llamafile/flags.cpp
index c36f208a..07a6ffdf 100644
--- a/llamafile/flags.cpp
+++ b/llamafile/flags.cpp
@@ -48,10 +48,10 @@ int FLAG_main_gpu = 0;
int FLAG_n_gpu_layers = -1;
int FLAG_seed = LLAMA_DEFAULT_SEED;
int FLAG_split_mode = LLAMA_SPLIT_MODE_LAYER;
-int FLAG_threads = 64;
+int FLAG_threads;
int FLAG_ubatch = 512;
int FLAG_verbose = 0;
-int FLAG_workers = 0;
+int FLAG_workers;
int cpu_get_num_math();
diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cpp
index b08b9f96..8aa34f3c 100644
--- a/llamafile/server/client.cpp
+++ b/llamafile/server/client.cpp
@@ -17,39 +17,25 @@
#include "client.h"
-#include <ctl/optional.h>
-#include <ctl/string.h>
#include <errno.h>
-#include <libc/fmt/itoa.h>
-#include <libc/str/slice.h>
#include <limits.h>
#include <string.h>
-#include <sys/auxv.h>
#include <sys/uio.h>
#include <time.h>
#include <unistd.h>
-#include "json.h"
#include "llama.cpp/llama.h"
#include "llamafile/llamafile.h"
#include "llamafile/version.h"
+
#include "log.h"
#include "time.h"
-#include "utils.h"
#define STANDARD_RESPONSE_HEADERS \
"Server: llamafile/" LLAMAFILE_VERSION_STRING "\r\n" \
"Referrer-Policy: origin\r\n" \
"Cache-Control: private; max-age=0\r\n"
-#define HasHeader(H) (!!msg.headers[H].a)
-#define HeaderData(H) (ibuf.p + msg.headers[H].a)
-#define HeaderLength(H) (msg.headers[H].b - msg.headers[H].a)
-#define HeaderEqual(H, S) \
- SlicesEqual(S, strlen(S), HeaderData(H), HeaderLength(H))
-#define HeaderEqualCase(H, S) \
- SlicesEqualCase(S, strlen(S), HeaderData(H), HeaderLength(H))
-
using namespace ctl;
Client::Client() : ibuf(8 * 1024 * 1024), obuf(64 * 1024 * 1024)
@@ -376,119 +362,3 @@ Client::param(string_view key)
string_view(url.params.p[i].val.p, url.params.p[i].val.n));
return {};
}
-
-static string_view
-or_empty(optional<string_view> x)
-{
- if (x.has_value())
- return x.value();
- return {};
-}
-
-bool
-Client::tokenize()
-{
- if (msg.method != kHttpGet && msg.method != kHttpPost)
- return send_error(405);
-
- if (!read_payload())
- return false;
-
- // get prompt
- //
- // 1. Allow GET "/tokenize?prompt=foo"
- // 2. Allow POST "prompt=foo" (application/x-www-form-urlencoded)
- // 3. Allow POST "foo" (text/plain)
- //
- string_view input;
- optional<string_view> prompt = param("prompt");
- if (prompt.has_value()) {
- input = prompt.value();
- } else if (HasHeader(kHttpContentType)) {
- if (IsMimeType(HeaderData(kHttpContentType),
- HeaderLength(kHttpContentType),
- "text/plain")) {
- input = payload;
- } else {
- return send_error(501, "Content Type Not Implemented");
- }
- } else {
- input = payload;
- }
-
- // get optional parameters
- bool add_special = atob(or_empty(param("add_special")), true);
- bool parse_special = atob(or_empty(param("parse_special")), false);
-
- // setup statistics
- rusage rustart = {};
- getrusage(RUSAGE_THREAD, &rustart);
- timespec started = timespec_real();
-
- // turn text into tokens
- extern llama_model* g_model;
- int maxcount = input.size() + 16;
- llama_token* toks = new llama_token[maxcount];
- int count = llama_tokenize(g_model,
- input.data(),
- input.size(),
- toks,
- maxcount,
- add_special,
- parse_special);
- if (count < 0) {
- delete[] toks;
- __builtin_trap();
- }
-
- // serialize tokens to json
- char* p = obuf.p;
- p = stpcpy(p, "{\r\n");
- p = stpcpy(p, " \"add_special\": ");
- p = encode_bool(p, add_special);
- p = stpcpy(p, ",\n");
- p = stpcpy(p, " \"parse_special\": ");
- p = encode_bool(p, parse_special);
- p = stpcpy(p, ",\n");
- p = stpcpy(p, " \"tokens\": [");
- for (int i = 0; i < count; ++i) {
- if (i)
- *p++ = ',';
- p = stpcpy(p, "\r\n ");
- char s[32];
- int n = llama_token_to_piece(g_model, toks[i], s, sizeof(s), true);
- if (n < 0) {
- delete[] toks;
- __builtin_trap();
- }
- p = encode_json(p, string_view(s, n));
- }
- p = stpcpy(p, "\r\n ]\r\n");
- p = stpcpy(p, "}\r\n");
- string_view content(obuf.p, p - obuf.p);
- delete[] toks;
-
- // collect statistics
- rusage ruend = {};
- getrusage(RUSAGE_THREAD, &ruend);
- timeval user = timeval_sub(ruend.ru_utime, rustart.ru_utime);
- timeval system = timeval_sub(ruend.ru_stime, rustart.ru_stime);
- timespec ended = timespec_real();
- timespec wall = timespec_sub(ended, started);
- long wall_us = timespec_tomicros(wall);
- long user_us = timeval_tomicros(user);
- long system_us = timeval_tomicros(system);
-
- // send response
- char* headers = p;
- p = start_response(p, 200);
- p = stpcpy(p, "Content-Type: application/json\r\n");
- p = stpcpy(p, "X-Wall-Micros: ");
- p = FormatInt64(p, wall_us);
- p = stpcpy(p, "\r\nX-User-Micros: ");
- p = FormatInt64(p, user_us);
- p = stpcpy(p, "\r\nX-System-Micros: ");
- p = FormatInt64(p, system_us);
- p = stpcpy(p, "\r\n");
- return send_response(headers, p, content);
-}
diff --git a/llamafile/server/client.h b/llamafile/server/client.h
index 2234c874..140d4b4a 100644
--- a/llamafile/server/client.h
+++ b/llamafile/server/client.h
@@ -20,12 +20,22 @@
#include "buffer.h"
#include <ctl/optional.h>
-#include <ctl/string_view.h>
+#include <ctl/string.h>
+#include <libc/fmt/itoa.h>
+#include <libc/str/slice.h>
#include <net/http/http.h>
#include <net/http/url.h>
#include <sys/resource.h>
#include <time.h>
+#define HasHeader(H) (!!msg.headers[H].a)
+#define HeaderData(H) (ibuf.p + msg.headers[H].a)
+#define HeaderLength(H) (msg.headers[H].b - msg.headers[H].a)
+#define HeaderEqual(H, S) \
+ SlicesEqual(S, strlen(S), HeaderData(H), HeaderLength(H))
+#define HeaderEqualCase(H, S) \
+ SlicesEqualCase(S, strlen(S), HeaderData(H), HeaderLength(H))
+
struct Client
{
int fd = -1;
diff --git a/llamafile/server/crash.cpp b/llamafile/server/crash.cpp
index 3aa9f2c8..afd8305f 100644
--- a/llamafile/server/crash.cpp
+++ b/llamafile/server/crash.cpp
@@ -18,7 +18,6 @@
#include "signals.h"
#include "utils.h"
-#include <assert.h>
#include <cosmo.h>
#include <ucontext.h>
@@ -50,14 +49,16 @@ is_call(const unsigned char* p)
return 0;
}
-void
+char*
describe_crash(char* buf, size_t len, int sig, siginfo_t* si, void* arg)
{
- unassert(len >= 64);
+ char* p = buf;
+
+ // check minimum length
+ if (len < 64)
+ return p;
// describe crash
- char* p = buf;
- char* pe = p + len;
char signame[21];
p = stpcpy(p, strsignal_r(sig, signame));
if (si && //
@@ -71,24 +72,31 @@ describe_crash(char* buf, size_t len, int sig, siginfo_t* si, void* arg)
}
// get stack frame daisy chain
- long pc;
+ StackFrame pc;
+ StackFrame* sf;
ucontext_t* ctx;
- struct StackFrame* sf;
if ((ctx = (ucontext_t*)arg)) {
- pc = ctx->uc_mcontext.PC;
- sf = (struct StackFrame*)ctx->uc_mcontext.BP;
+ pc.addr = ctx->uc_mcontext.PC;
+ pc.next = (struct StackFrame*)ctx->uc_mcontext.BP;
+ sf = &pc;
} else {
- pc = 0;
sf = (struct StackFrame*)__builtin_frame_address(0);
}
// describe backtrace
p = stpcpy(p, " bt ");
- if (pc) {
- p = hexcpy(p, pc);
- *p++ = ' ';
- }
+ p = describe_backtrace(p, len - (p - buf), sf);
+
+ return p;
+}
+
+char*
+describe_backtrace(char* p, size_t len, const StackFrame* sf)
+{
+ char* pe = p + len;
bool gotsome = false;
+
+ // show address of each function
while (sf) {
if (kisdangerous(sf)) {
if (p + 1 + 9 + 1 < pe) {
@@ -124,5 +132,7 @@ describe_crash(char* buf, size_t len, int sig, siginfo_t* si, void* arg)
}
// terminate string
- *p = '\0';
+ if (p < pe)
+ *p = '\0';
+ return p;
}
diff --git a/llamafile/server/listen.cpp b/llamafile/server/listen.cpp
index 608589da..2f71ff46 100644
--- a/llamafile/server/listen.cpp
+++ b/llamafile/server/listen.cpp
@@ -105,6 +105,7 @@ create_listening_socket(const char* hostport)
uint32_t* hostips;
for (hostips = GetHostIps(), i = 0; hostips[i]; ++i)
print_listening_url(hostips[i], ntohs(in->sin_port));
+ free(hostips);
} else {
print_listening_url(ntohl(in->sin_addr.s_addr), ntohs(in->sin_port));
}
diff --git a/llamafile/server/main.cpp b/llamafile/server/main.cpp
index 0a195b44..2979c568 100644
--- a/llamafile/server/main.cpp
+++ b/llamafile/server/main.cpp
@@ -16,7 +16,6 @@
// limitations under the License.
#include <assert.h>
-#include <third_party/dlmalloc/dlmalloc.h>
#include <tool/args/args.h>
#include "llama.cpp/llama.h"
@@ -29,6 +28,9 @@
#include "signals.h"
#include "time.h"
+extern "C" void
+_pthread_decimate(void);
+
Server* g_server;
llama_model* g_model;
@@ -65,7 +67,7 @@ main(int argc, char* argv[])
// create server
if (FLAG_workers <= 0)
- FLAG_workers = __get_cpu_count();
+ FLAG_workers = __get_cpu_count() + 4;
if (FLAG_workers <= 0)
FLAG_workers = 16;
set_thread_name("server");
@@ -74,9 +76,9 @@ main(int argc, char* argv[])
unassert(!g_server->spawn());
// run server
- setup_signals();
+ signals_init();
g_server->run();
- restore_signals();
+ signals_destroy();
// shutdown server
LOG("shutdown");
@@ -84,8 +86,11 @@ main(int argc, char* argv[])
g_server->close();
delete g_server;
llama_free_model(g_model);
+ time_destroy();
LOG("exit");
- // // quality assurance
- // CheckForMemoryLeaks();
+ // quality assurance
+ while (!pthread_orphan_np())
+ _pthread_decimate();
+ CheckForMemoryLeaks();
}
diff --git a/llamafile/server/memlog.inc b/llamafile/server/memlog.inc
new file mode 100644
index 00000000..57907a69
--- /dev/null
+++ b/llamafile/server/memlog.inc
@@ -0,0 +1,56 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "signals.h"
+#include <cosmo.h>
+#include <stdlib.h>
+#include <third_party/dlmalloc/dlmalloc.h>
+
+void*
+memlog(void* p)
+{
+ char bt[160];
+ StackFrame* sf;
+ sf = (StackFrame*)__builtin_frame_address(0);
+ describe_backtrace(bt, sizeof(bt), sf);
+ kprintf("alloc %p bt %s\n", p, bt);
+ return p;
+}
+
+void*
+malloc(size_t n)
+{
+ return memlog(dlmalloc(n));
+}
+
+void*
+calloc(size_t n, size_t z)
+{
+ return memlog(dlcalloc(n, z));
+}
+
+void*
+realloc(void* p, size_t n)
+{
+ return memlog(dlrealloc(p, n));
+}
+
+void*
+memalign(size_t a, size_t n)
+{
+ return memlog(dlmemalign(a, n));
+}
diff --git a/llamafile/server/server.cpp b/llamafile/server/server.cpp
index 3d3e8d98..83960637 100644
--- a/llamafile/server/server.cpp
+++ b/llamafile/server/server.cpp
@@ -162,7 +162,7 @@ Server::run()
if (terminated.load(std::memory_order_acquire))
break;
int missing =
- FLAG_threads - worker_count.load(std::memory_order_acquire);
+ FLAG_workers - worker_count.load(std::memory_order_acquire);
for (int i = 0; i < missing; ++i)
spawn();
}
diff --git a/llamafile/server/signals.cpp b/llamafile/server/signals.cpp
index f363ce5b..23932a6b 100644
--- a/llamafile/server/signals.cpp
+++ b/llamafile/server/signals.cpp
@@ -17,11 +17,27 @@
#include "signals.h"
+#include <signal.h>
#include <ucontext.h>
#include "log.h"
#include "server.h"
+static struct
+{
+ struct sigaction sigint; // ctrl-c
+ struct sigaction sighup; // terminal close
+ struct sigaction sigterm; // kill
+ struct sigaction sigabrt; // abort()
+ struct sigaction sigtrap; // breakpoint
+ struct sigaction sigfpe; // illegal math
+ struct sigaction sigbus; // illegal memory
+ struct sigaction sigsegv; // illegal memory
+ struct sigaction sigill; // illegal instruction
+ struct sigaction sigxcpu; // out of cpu quota
+ struct sigaction sigxfsz; // file too large
+} old;
+
void
on_termination_signal(int sig)
{
@@ -40,55 +56,49 @@ on_crash_signal(int sig, siginfo_t* si, void* arg)
}
void
-setup_signals(void)
+signals_init(void)
{
struct sigaction sa;
sa.sa_flags = SA_SIGINFO;
sigemptyset(&sa.sa_mask);
sa.sa_handler = on_termination_signal;
- sigaction(SIGINT, &sa, 0); // ctrl-c
- sigaction(SIGHUP, &sa, 0); // terminal close
- sigaction(SIGTERM, &sa, 0); // kill
+ sigaction(SIGINT, &sa, &old.sigint);
+ sigaction(SIGHUP, &sa, &old.sighup);
+ sigaction(SIGTERM, &sa, &old.sigterm);
sa.sa_sigaction = on_crash_signal;
- sigaddset(&sa.sa_mask, SIGABRT); // abort()
- sigaddset(&sa.sa_mask, SIGTRAP); // breakpoint
- sigaddset(&sa.sa_mask, SIGFPE); // illegal math
- sigaddset(&sa.sa_mask, SIGBUS); // illegal memory
- sigaddset(&sa.sa_mask, SIGSEGV); // illegal memory
- sigaddset(&sa.sa_mask, SIGILL); // illegal instruction
- sigaddset(&sa.sa_mask, SIGXCPU); // out of cpu quota
- sigaddset(&sa.sa_mask, SIGXFSZ); // file too large
+ sigaddset(&sa.sa_mask, SIGABRT);
+ sigaddset(&sa.sa_mask, SIGTRAP);
+ sigaddset(&sa.sa_mask, SIGFPE);
+ sigaddset(&sa.sa_mask, SIGBUS);
+ sigaddset(&sa.sa_mask, SIGSEGV);
+ sigaddset(&sa.sa_mask, SIGILL);
+ sigaddset(&sa.sa_mask, SIGXCPU);
+ sigaddset(&sa.sa_mask, SIGXFSZ);
- sigaction(SIGABRT, &sa, 0); // abort()
- sigaction(SIGTRAP, &sa, 0); // breakpoint
- sigaction(SIGFPE, &sa, 0); // illegal math
- sigaction(SIGBUS, &sa, 0); // illegal memory
- sigaction(SIGSEGV, &sa, 0); // illegal memory
- sigaction(SIGILL, &sa, 0); // illegal instruction
- sigaction(SIGXCPU, &sa, 0); // out of cpu quota
- sigaction(SIGXFSZ, &sa, 0); // file too large
+ sigaction(SIGABRT, &sa, &old.sigabrt);
+ sigaction(SIGTRAP, &sa, &old.sigtrap);
+ sigaction(SIGFPE, &sa, &old.sigfpe);
+ sigaction(SIGBUS, &sa, &old.sigbus);
+ sigaction(SIGSEGV, &sa, &old.sigsegv);
+ sigaction(SIGILL, &sa, &old.sigill);
+ sigaction(SIGXCPU, &sa, &old.sigxcpu);
+ sigaction(SIGXFSZ, &sa, &old.sigxfsz);
}
void
-restore_signals(void)
+signals_destroy(void)
{
- struct sigaction sa;
- sa.sa_flags = 0;
- sa.sa_handler = SIG_DFL;
- sigemptyset(&sa.sa_mask);
-
- sigaction(SIGINT, &sa, 0); // ctrl-c
- sigaction(SIGHUP, &sa, 0); // terminal close
- sigaction(SIGTERM, &sa, 0); // kill
-
- sigaction(SIGABRT, &sa, 0); // abort()
- sigaction(SIGTRAP, &sa, 0); // breakpoint
- sigaction(SIGFPE, &sa, 0); // illegal math
- sigaction(SIGBUS, &sa, 0); // illegal memory
- sigaction(SIGSEGV, &sa, 0); // illegal memory
- sigaction(SIGILL, &sa, 0); // illegal instruction
- sigaction(SIGXCPU, &sa, 0); // out of cpu quota
- sigaction(SIGXFSZ, &sa, 0); // file too large
+ sigaction(SIGINT, &old.sigint, 0);
+ sigaction(SIGHUP, &old.sighup, 0);
+ sigaction(SIGTERM, &old.sigterm, 0);
+ sigaction(SIGABRT, &old.sigabrt, 0);
+ sigaction(SIGTRAP, &old.sigtrap, 0);
+ sigaction(SIGFPE, &old.sigfpe, 0);
+ sigaction(SIGBUS, &old.sigbus, 0);
+ sigaction(SIGSEGV, &old.sigsegv, 0);
+ sigaction(SIGILL, &old.sigill, 0);
+ sigaction(SIGXCPU, &old.sigxcpu, 0);
+ sigaction(SIGXFSZ, &old.sigxfsz, 0);
}
diff --git a/llamafile/server/signals.h b/llamafile/server/signals.h
index ab9104fe..6a196b34 100644
--- a/llamafile/server/signals.h
+++ b/llamafile/server/signals.h
@@ -18,11 +18,16 @@
#pragma once
#include <signal.h>
-void
-setup_signals(void);
+struct StackFrame;
void
-restore_signals(void);
+signals_init(void);
void
+signals_destroy(void);
+
+char*
describe_crash(char*, size_t, int, siginfo_t*, void*);
+
+char*
+describe_backtrace(char*, size_t, const StackFrame*);
diff --git a/llamafile/server/time.cpp b/llamafile/server/time.cpp
index c733f80f..de3a4895 100644
--- a/llamafile/server/time.cpp
+++ b/llamafile/server/time.cpp
@@ -37,6 +37,7 @@ struct Clock
};
static Clock g_clck[2];
+static pthread_t g_time_thread;
static void
set_clck(Clock* clck, long time, long date)
@@ -131,7 +132,7 @@ time_worker(void* arg)
sigaddset(&ss, SIGTERM);
sigaddset(&ss, SIGUSR1);
sigaddset(&ss, SIGALRM);
- pthread_sigmask(SIG_BLOCK, &ss, 0);
+ pthread_sigmask(SIG_SETMASK, &ss, 0);
set_thread_name("localtime");
for (;;) {
sleep(10);
@@ -144,17 +145,25 @@ void
time_init()
{
update_time();
- pthread_t th;
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, 65536);
pthread_attr_setguardsize(&attr, getauxval(AT_PAGESZ));
- pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
- if (pthread_create(&th, &attr, time_worker, 0))
+ if (pthread_create(&g_time_thread, &attr, time_worker, 0))
__builtin_trap();
pthread_attr_destroy(&attr);
}
+void
+time_destroy()
+{
+ if (!g_time_thread)
+ return;
+ pthread_cancel(g_time_thread);
+ if (pthread_join(g_time_thread, 0))
+ __builtin_trap();
+}
+
static const char kMonDays[2][12] = {
{ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },
{ 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },
diff --git a/llamafile/server/time.h b/llamafile/server/time.h
index 3ebe77e7..d23a5f51 100644
--- a/llamafile/server/time.h
+++ b/llamafile/server/time.h
@@ -21,6 +21,9 @@
void
time_init();
+void
+time_destroy();
+
void
gmtime_lockless(long, tm*);
diff --git a/llamafile/server/tokenize.cpp b/llamafile/server/tokenize.cpp
new file mode 100644
index 00000000..83a2aff8
--- /dev/null
+++ b/llamafile/server/tokenize.cpp
@@ -0,0 +1,144 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "client.h"
+
+#include <string.h>
+#include <sys/resource.h>
+
+#include "llama.cpp/llama.h"
+
+#include "json.h"
+#include "utils.h"
+
+using namespace ctl;
+
+static string_view
+or_empty(optional<string_view> x)
+{
+ if (x.has_value())
+ return x.value();
+ return {};
+}
+
+bool
+Client::tokenize()
+{
+ if (msg.method != kHttpGet && msg.method != kHttpPost)
+ return send_error(405);
+
+ if (!read_payload())
+ return false;
+
+ // get prompt
+ //
+ // 1. Allow GET "/tokenize?prompt=foo"
+ // 2. Allow POST "prompt=foo" (application/x-www-form-urlencoded)
+ // 3. Allow POST "foo" (text/plain)
+ //
+ string_view input;
+ optional<string_view> prompt = param("prompt");
+ if (prompt.has_value()) {
+ input = prompt.value();
+ } else if (HasHeader(kHttpContentType)) {
+ if (IsMimeType(HeaderData(kHttpContentType),
+ HeaderLength(kHttpContentType),
+ "text/plain")) {
+ input = payload;
+ } else {
+ return send_error(501, "Content Type Not Implemented");
+ }
+ } else {
+ input = payload;
+ }
+
+ // get optional parameters
+ bool add_special = atob(or_empty(param("add_special")), true);
+ bool parse_special = atob(or_empty(param("parse_special")), false);
+
+ // setup statistics
+ rusage rustart = {};
+ getrusage(RUSAGE_THREAD, &rustart);
+ timespec started = timespec_real();
+
+ // turn text into tokens
+ extern llama_model* g_model;
+ int maxcount = input.size() + 16;
+ llama_token* toks = new llama_token[maxcount];
+ int count = llama_tokenize(g_model,
+ input.data(),
+ input.size(),
+ toks,
+ maxcount,
+ add_special,
+ parse_special);
+ if (count < 0) {
+ delete[] toks;
+ __builtin_trap();
+ }
+
+ // serialize tokens to json
+ char* p = obuf.p;
+ p = stpcpy(p, "{\r\n");
+ p = stpcpy(p, " \"add_special\": ");
+ p = encode_bool(p, add_special);
+ p = stpcpy(p, ",\n");
+ p = stpcpy(p, " \"parse_special\": ");
+ p = encode_bool(p, parse_special);
+ p = stpcpy(p, ",\n");
+ p = stpcpy(p, " \"tokens\": [");
+ for (int i = 0; i < count; ++i) {
+ if (i)
+ *p++ = ',';
+ p = stpcpy(p, "\r\n ");
+ char s[32];
+ int n = llama_token_to_piece(g_model, toks[i], s, sizeof(s), true);
+ if (n < 0) {
+ delete[] toks;
+ __builtin_trap();
+ }
+ p = encode_json(p, string_view(s, n));
+ }
+ p = stpcpy(p, "\r\n ]\r\n");
+ p = stpcpy(p, "}\r\n");
+ string_view content(obuf.p, p - obuf.p);
+ delete[] toks;
+
+ // collect statistics
+ rusage ruend = {};
+ getrusage(RUSAGE_THREAD, &ruend);
+ timeval user = timeval_sub(ruend.ru_utime, rustart.ru_utime);
+ timeval system = timeval_sub(ruend.ru_stime, rustart.ru_stime);
+ timespec ended = timespec_real();
+ timespec wall = timespec_sub(ended, started);
+ long wall_us = timespec_tomicros(wall);
+ long user_us = timeval_tomicros(user);
+ long system_us = timeval_tomicros(system);
+
+ // send response
+ char* headers = p;
+ p = start_response(p, 200);
+ p = stpcpy(p, "Content-Type: application/json\r\n");
+ p = stpcpy(p, "X-Wall-Micros: ");
+ p = FormatInt64(p, wall_us);
+ p = stpcpy(p, "\r\nX-User-Micros: ");
+ p = FormatInt64(p, user_us);
+ p = stpcpy(p, "\r\nX-System-Micros: ");
+ p = FormatInt64(p, system_us);
+ p = stpcpy(p, "\r\n");
+ return send_response(headers, p, content);
+}
diff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cpp
index e9746501..1d3a0ecd 100644
--- a/llamafile/server/worker.cpp
+++ b/llamafile/server/worker.cpp
@@ -132,7 +132,7 @@ Worker::run()
sigaddset(&mask, SIGTERM);
sigaddset(&mask, SIGUSR1);
sigaddset(&mask, SIGALRM);
- pthread_sigmask(SIG_BLOCK, &mask, 0);
+ pthread_sigmask(SIG_SETMASK, &mask, 0);
handle();
}