Skip to content

Distribute and run LLMs with a single file.

Revert recent upstream developments on server

Justine Tunney8ff1283d7b35
diff --git a/llama.cpp/server/httplib.h b/llama.cpp/server/httplib.h
index f360bd93..28746000 100644
--- a/llama.cpp/server/httplib.h
+++ b/llama.cpp/server/httplib.h
@@ -1,14 +1,14 @@
//
// httplib.h
//
-// Copyright (c) 2024 Yuji Hirose. All rights reserved.
+// Copyright (c) 2023 Yuji Hirose. All rights reserved.
// MIT License
//
#ifndef CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_HTTPLIB_H
-#define CPPHTTPLIB_VERSION "0.15.3"
+#define CPPHTTPLIB_VERSION "0.12.2"
/*
* Configuration
@@ -82,10 +82,6 @@
#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192
#endif
-#ifndef CPPHTTPLIB_RANGE_MAX_COUNT
-#define CPPHTTPLIB_RANGE_MAX_COUNT 1024
-#endif
-
#ifndef CPPHTTPLIB_TCP_NODELAY
#define CPPHTTPLIB_TCP_NODELAY false
#endif
@@ -164,6 +160,10 @@ using ssize_t = long;
#define WSA_FLAG_NO_HANDLE_INHERIT 0x80
#endif
+#ifndef strcasecmp
+#define strcasecmp _stricmp
+#endif // strcasecmp
+
using socket_t = SOCKET;
#ifdef CPPHTTPLIB_USE_POLL
#define poll(fds, nfds, timeout) WSAPoll(fds, nfds, timeout)
@@ -172,15 +172,9 @@ using socket_t = SOCKET;
#else // not _WIN32
#include <arpa/inet.h>
-#if !defined(_AIX) && !defined(__MVS__)
+#ifndef _AIX
#include <ifaddrs.h>
#endif
-#ifdef __MVS__
-#include <strings.h>
-#ifndef NI_MAXHOST
-#define NI_MAXHOST 1025
-#endif
-#endif
#include <net/if.h>
#include <netdb.h>
#include <netinet/in.h>
@@ -193,7 +187,6 @@ using socket_t = SOCKET;
#endif
#include <csignal>
#include <pthread.h>
-#include <sys/mman.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <sys/un.h>
@@ -214,7 +207,6 @@ using socket_t = int;
#include <condition_variable>
#include <cstring>
#include <errno.h>
-#include <exception>
#include <fcntl.h>
#include <fstream>
#include <functional>
@@ -231,9 +223,6 @@ using socket_t = int;
#include <string>
#include <sys/stat.h>
#include <thread>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
#ifdef _WIN32
@@ -248,6 +237,7 @@ using socket_t = int;
#ifdef _MSC_VER
#pragma comment(lib, "crypt32.lib")
+#pragma comment(lib, "cryptui.lib")
#endif
#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)
#include <TargetConditionals.h>
@@ -269,8 +259,10 @@ using socket_t = int;
#include <iostream>
#include <sstream>
-#if OPENSSL_VERSION_NUMBER < 0x30000000L
-#error Sorry, OpenSSL versions prior to 3.0.0 are not supported
+#if OPENSSL_VERSION_NUMBER < 0x1010100fL
+#error Sorry, OpenSSL versions prior to 1.1.1 are not supported
+#elif OPENSSL_VERSION_NUMBER < 0x30000000L
+#define SSL_get1_peer_certificate SSL_get_peer_certificate
#endif
#endif
@@ -329,7 +321,7 @@ struct scope_exit {
explicit scope_exit(std::function<void(void)> &&f)
: exit_function(std::move(f)), execute_on_destruction{true} {}
- scope_exit(scope_exit &&rhs) noexcept
+ scope_exit(scope_exit &&rhs)
: exit_function(std::move(rhs.exit_function)),
execute_on_destruction{rhs.execute_on_destruction} {
rhs.release();
@@ -352,81 +344,6 @@ private:
} // namespace detail
-enum StatusCode {
- // Information responses
- Continue_100 = 100,
- SwitchingProtocol_101 = 101,
- Processing_102 = 102,
- EarlyHints_103 = 103,
-
- // Successful responses
- OK_200 = 200,
- Created_201 = 201,
- Accepted_202 = 202,
- NonAuthoritativeInformation_203 = 203,
- NoContent_204 = 204,
- ResetContent_205 = 205,
- PartialContent_206 = 206,
- MultiStatus_207 = 207,
- AlreadyReported_208 = 208,
- IMUsed_226 = 226,
-
- // Redirection messages
- MultipleChoices_300 = 300,
- MovedPermanently_301 = 301,
- Found_302 = 302,
- SeeOther_303 = 303,
- NotModified_304 = 304,
- UseProxy_305 = 305,
- unused_306 = 306,
- TemporaryRedirect_307 = 307,
- PermanentRedirect_308 = 308,
-
- // Client error responses
- BadRequest_400 = 400,
- Unauthorized_401 = 401,
- PaymentRequired_402 = 402,
- Forbidden_403 = 403,
- NotFound_404 = 404,
- MethodNotAllowed_405 = 405,
- NotAcceptable_406 = 406,
- ProxyAuthenticationRequired_407 = 407,
- RequestTimeout_408 = 408,
- Conflict_409 = 409,
- Gone_410 = 410,
- LengthRequired_411 = 411,
- PreconditionFailed_412 = 412,
- PayloadTooLarge_413 = 413,
- UriTooLong_414 = 414,
- UnsupportedMediaType_415 = 415,
- RangeNotSatisfiable_416 = 416,
- ExpectationFailed_417 = 417,
- ImATeapot_418 = 418,
- MisdirectedRequest_421 = 421,
- UnprocessableContent_422 = 422,
- Locked_423 = 423,
- FailedDependency_424 = 424,
- TooEarly_425 = 425,
- UpgradeRequired_426 = 426,
- PreconditionRequired_428 = 428,
- TooManyRequests_429 = 429,
- RequestHeaderFieldsTooLarge_431 = 431,
- UnavailableForLegalReasons_451 = 451,
-
- // Server error responses
- InternalServerError_500 = 500,
- NotImplemented_501 = 501,
- BadGateway_502 = 502,
- ServiceUnavailable_503 = 503,
- GatewayTimeout_504 = 504,
- HttpVersionNotSupported_505 = 505,
- VariantAlsoNegotiates_506 = 506,
- InsufficientStorage_507 = 507,
- LoopDetected_508 = 508,
- NotExtended_510 = 510,
- NetworkAuthenticationRequired_511 = 511,
-};
-
using Headers = std::multimap<std::string, std::string, detail::ci>;
using Params = std::multimap<std::string, std::string>;
@@ -456,18 +373,17 @@ public:
DataSink &operator=(DataSink &&) = delete;
std::function<bool(const char *data, size_t data_len)> write;
- std::function<bool()> is_writable;
std::function<void()> done;
std::function<void(const Headers &trailer)> done_with_trailer;
std::ostream os;
private:
- class data_sink_streambuf final : public std::streambuf {
+ class data_sink_streambuf : public std::streambuf {
public:
explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {}
protected:
- std::streamsize xsputn(const char *s, std::streamsize n) override {
+ std::streamsize xsputn(const char *s, std::streamsize n) {
sink_.write(s, static_cast<size_t>(n));
return n;
}
@@ -549,7 +465,6 @@ struct Request {
MultipartFormDataMap files;
Ranges ranges;
Match matches;
- std::unordered_map<std::string, std::string> path_params;
// for client
ResponseHandler response_handler;
@@ -561,7 +476,8 @@ struct Request {
bool has_header(const std::string &key) const;
std::string get_header_value(const std::string &key, size_t id = 0) const;
- uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const;
+ template <typename T>
+ T get_header_value(const std::string &key, size_t id = 0) const;
size_t get_header_value_count(const std::string &key) const;
void set_header(const std::string &key, const std::string &val);
@@ -593,14 +509,14 @@ struct Response {
bool has_header(const std::string &key) const;
std::string get_header_value(const std::string &key, size_t id = 0) const;
- uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const;
+ template <typename T>
+ T get_header_value(const std::string &key, size_t id = 0) const;
size_t get_header_value_count(const std::string &key) const;
void set_header(const std::string &key, const std::string &val);
- void set_redirect(const std::string &url, int status = StatusCode::Found_302);
+ void set_redirect(const std::string &url, int status = 302);
void set_content(const char *s, size_t n, const std::string &content_type);
void set_content(const std::string &s, const std::string &content_type);
- void set_content(std::string &&s, const std::string &content_type);
void set_content_provider(
size_t length, const std::string &content_type, ContentProvider provider,
@@ -657,16 +573,15 @@ public:
TaskQueue() = default;
virtual ~TaskQueue() = default;
- virtual bool enqueue(std::function<void()> fn) = 0;
+ virtual void enqueue(std::function<void()> fn) = 0;
virtual void shutdown() = 0;
virtual void on_idle() {}
};
-class ThreadPool final : public TaskQueue {
+class ThreadPool : public TaskQueue {
public:
- explicit ThreadPool(size_t n, size_t mqr = 0)
- : shutdown_(false), max_queued_requests_(mqr) {
+ explicit ThreadPool(size_t n) : shutdown_(false) {
while (n) {
threads_.emplace_back(worker(*this));
n--;
@@ -676,17 +591,13 @@ public:
ThreadPool(const ThreadPool &) = delete;
~ThreadPool() override = default;
- bool enqueue(std::function<void()> fn) override {
+ void enqueue(std::function<void()> fn) override {
{
std::unique_lock<std::mutex> lock(mutex_);
- if (max_queued_requests_ > 0 && jobs_.size() >= max_queued_requests_) {
- return false;
- }
jobs_.push_back(std::move(fn));
}
cond_.notify_one();
- return true;
}
void shutdown() override {
@@ -736,7 +647,6 @@ private:
std::list<std::function<void()>> jobs_;
bool shutdown_;
- size_t max_queued_requests_ = 0;
std::condition_variable cond_;
std::mutex mutex_;
@@ -748,82 +658,6 @@ using SocketOptions = std::function<void(socket_t sock)>;
void default_socket_options(socket_t sock);
-const char *status_message(int status);
-
-std::string get_bearer_token_auth(const Request &req);
-
-namespace detail {
-
-class MatcherBase {
-public:
- virtual ~MatcherBase() = default;
-
- // Match request path and populate its matches and
- virtual bool match(Request &request) const = 0;
-};
-
-/**
- * Captures parameters in request path and stores them in Request::path_params
- *
- * Capture name is a substring of a pattern from : to /.
- * The rest of the pattern is matched agains the request path directly
- * Parameters are captured starting from the next character after
- * the end of the last matched static pattern fragment until the next /.
- *
- * Example pattern:
- * "/path/fragments/:capture/more/fragments/:second_capture"
- * Static fragments:
- * "/path/fragments/", "more/fragments/"
- *
- * Given the following request path:
- * "/path/fragments/:1/more/fragments/:2"
- * the resulting capture will be
- * {{"capture", "1"}, {"second_capture", "2"}}
- */
-class PathParamsMatcher final : public MatcherBase {
-public:
- PathParamsMatcher(const std::string &pattern);
-
- bool match(Request &request) const override;
-
-private:
- static constexpr char marker = ':';
- // Treat segment separators as the end of path parameter capture
- // Does not need to handle query parameters as they are parsed before path
- // matching
- static constexpr char separator = '/';
-
- // Contains static path fragments to match against, excluding the '/' after
- // path params
- // Fragments are separated by path params
- std::vector<std::string> static_fragments_;
- // Stores the names of the path parameters to be used as keys in the
- // Request::path_params map
- std::vector<std::string> param_names_;
-};
-
-/**
- * Performs std::regex_match on request path
- * and stores the result in Request::matches
- *
- * Note that regex match is performed directly on the whole request.
- * This means that wildcard patterns may match multiple path segments with /:
- * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end".
- */
-class RegexMatcher final : public MatcherBase {
-public:
- RegexMatcher(const std::string &pattern) : regex_(pattern) {}
-
- bool match(Request &request) const override;
-
-private:
- std::regex regex_;
-};
-
-ssize_t write_headers(Stream &strm, const Headers &headers);
-
-} // namespace detail
-
class Server {
public:
using Handler = std::function<void(const Request &, Response &)>;
@@ -868,7 +702,6 @@ public:
bool remove_mount_point(const std::string &mount_point);
Server &set_file_extension_and_mimetype_mapping(const std::string &ext,
const std::string &mime);
- Server &set_default_file_mimetype(const std::string &mime);
Server &set_file_request_handler(Handler handler);
Server &set_error_handler(HandlerWithResponse handler);
@@ -885,8 +718,6 @@ public:
Server &set_socket_options(SocketOptions socket_options);
Server &set_default_headers(Headers headers);
- Server &
- set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);
Server &set_keep_alive_max_count(size_t count);
Server &set_keep_alive_timeout(time_t sec);
@@ -934,14 +765,9 @@ protected:
size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH;
private:
- using Handlers =
- std::vector<std::pair<std::unique_ptr<detail::MatcherBase>, Handler>>;
+ using Handlers = std::vector<std::pair<std::regex, Handler>>;
using HandlersForContentReader =
- std::vector<std::pair<std::unique_ptr<detail::MatcherBase>,
- HandlerWithContentReader>>;
-
- static std::unique_ptr<detail::MatcherBase>
- make_matcher(const std::string &pattern);
+ std::vector<std::pair<std::regex, HandlerWithContentReader>>;
socket_t create_server_socket(const std::string &host, int port,
int socket_flags,
@@ -952,16 +778,16 @@ private:
bool routing(Request &req, Response &res, Stream &strm);
bool handle_file_request(const Request &req, Response &res,
bool head = false);
- bool dispatch_request(Request &req, Response &res,
- const Handlers &handlers) const;
- bool dispatch_request_for_content_reader(
- Request &req, Response &res, ContentReader content_reader,
- const HandlersForContentReader &handlers) const;
+ bool dispatch_request(Request &req, Response &res, const Handlers &handlers);
+ bool
+ dispatch_request_for_content_reader(Request &req, Response &res,
+ ContentReader content_reader,
+ const HandlersForContentReader &handlers);
- bool parse_request_line(const char *s, Request &req) const;
+ bool parse_request_line(const char *s, Request &req);
void apply_ranges(const Request &req, Response &res,
- std::string &content_type, std::string &boundary) const;
- bool write_response(Stream &strm, bool close_connection, Request &req,
+ std::string &content_type, std::string &boundary);
+ bool write_response(Stream &strm, bool close_connection, const Request &req,
Response &res);
bool write_response_with_content(Stream &strm, bool close_connection,
const Request &req, Response &res);
@@ -980,23 +806,21 @@ private:
bool read_content_core(Stream &strm, Request &req, Response &res,
ContentReceiver receiver,
MultipartContentHeader multipart_header,
- ContentReceiver multipart_receiver) const;
+ ContentReceiver multipart_receiver);
virtual bool process_and_close_socket(socket_t sock);
- std::atomic<bool> is_running_{false};
- std::atomic<bool> done_{false};
-
struct MountPointEntry {
std::string mount_point;
std::string base_dir;
Headers headers;
};
std::vector<MountPointEntry> base_dirs_;
+
+ std::atomic<bool> is_running_{false};
+ std::atomic<bool> done_{false};
std::map<std::string, std::string> file_extension_and_mimetype_map_;
- std::string default_file_mimetype_ = "application/octet-stream";
Handler file_request_handler_;
-
Handlers get_handlers_;
Handlers post_handlers_;
HandlersForContentReader post_handlers_for_content_reader_;
@@ -1007,22 +831,18 @@ private:
Handlers delete_handlers_;
HandlersForContentReader delete_handlers_for_content_reader_;
Handlers options_handlers_;
-
HandlerWithResponse error_handler_;
ExceptionHandler exception_handler_;
HandlerWithResponse pre_routing_handler_;
Handler post_routing_handler_;
- Expect100ContinueHandler expect_100_continue_handler_;
-
Logger logger_;
+ Expect100ContinueHandler expect_100_continue_handler_;
int address_family_ = AF_UNSPEC;
bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY;
SocketOptions socket_options_ = default_socket_options;
Headers default_headers_;
- std::function<ssize_t(Stream &, Headers &)> header_writer_ =
- detail::write_headers;
};
enum class Error {
@@ -1040,19 +860,17 @@ enum class Error {
UnsupportedMultipartBoundaryChars,
Compression,
ConnectionTimeout,
- ProxyConnection,
// For internal use only
SSLPeerCouldBeClosed_,
};
-std::string to_string(Error error);
+std::string to_string(const Error error);
std::ostream &operator<<(std::ostream &os, const Error &obj);
class Result {
public:
- Result() = default;
Result(std::unique_ptr<Response> &&res, Error err,
Headers &&request_headers = Headers{})
: res_(std::move(res)), err_(err),
@@ -1075,13 +893,13 @@ public:
bool has_request_header(const std::string &key) const;
std::string get_request_header_value(const std::string &key,
size_t id = 0) const;
- uint64_t get_request_header_value_u64(const std::string &key,
- size_t id = 0) const;
+ template <typename T>
+ T get_request_header_value(const std::string &key, size_t id = 0) const;
size_t get_request_header_value_count(const std::string &key) const;
private:
std::unique_ptr<Response> res_;
- Error err_ = Error::Unknown;
+ Error err_;
Headers request_headers_;
};
@@ -1241,21 +1059,16 @@ public:
bool send(Request &req, Response &res, Error &error);
Result send(const Request &req);
- void stop();
-
- std::string host() const;
- int port() const;
-
size_t is_socket_open() const;
+
socket_t socket() const;
+ void stop();
+
void set_hostname_addr_map(std::map<std::string, std::string> addr_map);
void set_default_headers(Headers headers);
- void
- set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);
-
void set_address_family(int family);
void set_tcp_nodelay(bool on);
void set_socket_options(SocketOptions socket_options);
@@ -1304,7 +1117,6 @@ public:
void set_ca_cert_path(const std::string &ca_cert_file_path,
const std::string &ca_cert_dir_path = std::string());
void set_ca_cert_store(X509_STORE *ca_cert_store);
- X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size) const;
#endif
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
@@ -1333,14 +1145,14 @@ protected:
// Also, shutdown_ssl and close_socket should also NOT be called concurrently
// with a DIFFERENT thread sending requests using that socket.
virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully);
- void shutdown_socket(Socket &socket) const;
+ void shutdown_socket(Socket &socket);
void close_socket(Socket &socket);
bool process_request(Stream &strm, Request &req, Response &res,
bool close_connection, Error &error);
bool write_content_with_provider(Stream &strm, const Request &req,
- Error &error) const;
+ Error &error);
void copy_settings(const ClientImpl &rhs);
@@ -1365,10 +1177,6 @@ protected:
// Default headers
Headers default_headers_;
- // Header writer
- std::function<ssize_t(Stream &, Headers &)> header_writer_ =
- detail::write_headers;
-
// Settings
std::string client_cert_path_;
std::string client_key_path_;
@@ -1431,8 +1239,7 @@ private:
Result send_(Request &&req);
socket_t create_client_socket(Error &error) const;
- bool read_response_line(Stream &strm, const Request &req,
- Response &res) const;
+ bool read_response_line(Stream &strm, const Request &req, Response &res);
bool write_request(Stream &strm, Request &req, bool close_connection,
Error &error);
bool redirect(Request &req, Response &res, Error &error);
@@ -1451,7 +1258,7 @@ private:
const std::string &content_type);
ContentProviderWithoutLength get_multipart_content_provider(
const std::string &boundary, const MultipartFormDataItems &items,
- const MultipartFormDataProviderItems &provider_items) const;
+ const MultipartFormDataProviderItems &provider_items);
std::string adjust_host_string(const std::string &host) const;
@@ -1624,21 +1431,16 @@ public:
bool send(Request &req, Response &res, Error &error);
Result send(const Request &req);
- void stop();
-
- std::string host() const;
- int port() const;
-
size_t is_socket_open() const;
+
socket_t socket() const;
+ void stop();
+
void set_hostname_addr_map(std::map<std::string, std::string> addr_map);
void set_default_headers(Headers headers);
- void
- set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);
-
void set_address_family(int family);
void set_tcp_nodelay(bool on);
void set_socket_options(SocketOptions socket_options);
@@ -1695,7 +1497,6 @@ public:
const std::string &ca_cert_dir_path = std::string());
void set_ca_cert_store(X509_STORE *ca_cert_store);
- void load_ca_cert_store(const char *ca_cert, std::size_t size);
long get_openssl_verify_result() const;
@@ -1737,7 +1538,7 @@ private:
std::mutex ctx_mutex_;
};
-class SSLClient final : public ClientImpl {
+class SSLClient : public ClientImpl {
public:
explicit SSLClient(const std::string &host);
@@ -1745,19 +1546,16 @@ public:
explicit SSLClient(const std::string &host, int port,
const std::string &client_cert_path,
- const std::string &client_key_path,
- const std::string &private_key_password = std::string());
+ const std::string &client_key_path);
explicit SSLClient(const std::string &host, int port, X509 *client_cert,
- EVP_PKEY *client_key,
- const std::string &private_key_password = std::string());
+ EVP_PKEY *client_key);
~SSLClient() override;
bool is_valid() const override;
void set_ca_cert_store(X509_STORE *ca_cert_store);
- void load_ca_cert_store(const char *ca_cert, std::size_t size);
long get_openssl_verify_result() const;
@@ -1766,7 +1564,7 @@ public:
private:
bool create_and_connect_socket(Socket &socket, Error &error) override;
void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override;
- void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully);
+ void shutdown_ssl_impl(Socket &socket, bool shutdown_socket);
bool process_socket(const Socket &socket,
std::function<bool(Stream &strm)> callback) override;
@@ -1810,9 +1608,15 @@ inline void duration_to_sec_and_usec(const T &duration, U callback) {
callback(static_cast<time_t>(sec), static_cast<time_t>(usec));
}
-inline uint64_t get_header_value_u64(const Headers &headers,
- const std::string &key, size_t id,
- uint64_t def) {
+template <typename T>
+inline T get_header_value(const Headers & /*headers*/,
+ const std::string & /*key*/, size_t /*id*/ = 0,
+ uint64_t /*def*/ = 0) {}
+
+template <>
+inline uint64_t get_header_value<uint64_t>(const Headers &headers,
+ const std::string &key, size_t id,
+ uint64_t def) {
auto rng = headers.equal_range(key);
auto it = rng.first;
std::advance(it, static_cast<ssize_t>(id));
@@ -1824,14 +1628,14 @@ inline uint64_t get_header_value_u64(const Headers &headers,
} // namespace detail
-inline uint64_t Request::get_header_value_u64(const std::string &key,
- size_t id) const {
- return detail::get_header_value_u64(headers, key, id, 0);
+template <typename T>
+inline T Request::get_header_value(const std::string &key, size_t id) const {
+ return detail::get_header_value<T>(headers, key, id, 0);
}
-inline uint64_t Response::get_header_value_u64(const std::string &key,
- size_t id) const {
- return detail::get_header_value_u64(headers, key, id, 0);
+template <typename T>
+inline T Response::get_header_value(const std::string &key, size_t id) const {
+ return detail::get_header_value<T>(headers, key, id, 0);
}
template <typename... Args>
@@ -1861,106 +1665,21 @@ inline ssize_t Stream::write_format(const char *fmt, const Args &...args) {
inline void default_socket_options(socket_t sock) {
int yes = 1;
#ifdef _WIN32
- setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
- reinterpret_cast<const char *>(&yes), sizeof(yes));
+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char *>(&yes),
+ sizeof(yes));
setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,
- reinterpret_cast<const char *>(&yes), sizeof(yes));
+ reinterpret_cast<char *>(&yes), sizeof(yes));
#else
#ifdef SO_REUSEPORT
- setsockopt(sock, SOL_SOCKET, SO_REUSEPORT,
- reinterpret_cast<const void *>(&yes), sizeof(yes));
+ setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast<void *>(&yes),
+ sizeof(yes));
#else
- setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
- reinterpret_cast<const void *>(&yes), sizeof(yes));
+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<void *>(&yes),
+ sizeof(yes));
#endif
#endif
}
-inline const char *status_message(int status) {
- switch (status) {
- case StatusCode::Continue_100: return "Continue";
- case StatusCode::SwitchingProtocol_101: return "Switching Protocol";
- case StatusCode::Processing_102: return "Processing";
- case StatusCode::EarlyHints_103: return "Early Hints";
- case StatusCode::OK_200: return "OK";
- case StatusCode::Created_201: return "Created";
- case StatusCode::Accepted_202: return "Accepted";
- case StatusCode::NonAuthoritativeInformation_203:
- return "Non-Authoritative Information";
- case StatusCode::NoContent_204: return "No Content";
- case StatusCode::ResetContent_205: return "Reset Content";
- case StatusCode::PartialContent_206: return "Partial Content";
- case StatusCode::MultiStatus_207: return "Multi-Status";
- case StatusCode::AlreadyReported_208: return "Already Reported";
- case StatusCode::IMUsed_226: return "IM Used";
- case StatusCode::MultipleChoices_300: return "Multiple Choices";
- case StatusCode::MovedPermanently_301: return "Moved Permanently";
- case StatusCode::Found_302: return "Found";
- case StatusCode::SeeOther_303: return "See Other";
- case StatusCode::NotModified_304: return "Not Modified";
- case StatusCode::UseProxy_305: return "Use Proxy";
- case StatusCode::unused_306: return "unused";
- case StatusCode::TemporaryRedirect_307: return "Temporary Redirect";
- case StatusCode::PermanentRedirect_308: return "Permanent Redirect";
- case StatusCode::BadRequest_400: return "Bad Request";
- case StatusCode::Unauthorized_401: return "Unauthorized";
- case StatusCode::PaymentRequired_402: return "Payment Required";
- case StatusCode::Forbidden_403: return "Forbidden";
- case StatusCode::NotFound_404: return "Not Found";
- case StatusCode::MethodNotAllowed_405: return "Method Not Allowed";
- case StatusCode::NotAcceptable_406: return "Not Acceptable";
- case StatusCode::ProxyAuthenticationRequired_407:
- return "Proxy Authentication Required";
- case StatusCode::RequestTimeout_408: return "Request Timeout";
- case StatusCode::Conflict_409: return "Conflict";
- case StatusCode::Gone_410: return "Gone";
- case StatusCode::LengthRequired_411: return "Length Required";
- case StatusCode::PreconditionFailed_412: return "Precondition Failed";
- case StatusCode::PayloadTooLarge_413: return "Payload Too Large";
- case StatusCode::UriTooLong_414: return "URI Too Long";
- case StatusCode::UnsupportedMediaType_415: return "Unsupported Media Type";
- case StatusCode::RangeNotSatisfiable_416: return "Range Not Satisfiable";
- case StatusCode::ExpectationFailed_417: return "Expectation Failed";
- case StatusCode::ImATeapot_418: return "I'm a teapot";
- case StatusCode::MisdirectedRequest_421: return "Misdirected Request";
- case StatusCode::UnprocessableContent_422: return "Unprocessable Content";
- case StatusCode::Locked_423: return "Locked";
- case StatusCode::FailedDependency_424: return "Failed Dependency";
- case StatusCode::TooEarly_425: return "Too Early";
- case StatusCode::UpgradeRequired_426: return "Upgrade Required";
- case StatusCode::PreconditionRequired_428: return "Precondition Required";
- case StatusCode::TooManyRequests_429: return "Too Many Requests";
- case StatusCode::RequestHeaderFieldsTooLarge_431:
- return "Request Header Fields Too Large";
- case StatusCode::UnavailableForLegalReasons_451:
- return "Unavailable For Legal Reasons";
- case StatusCode::NotImplemented_501: return "Not Implemented";
- case StatusCode::BadGateway_502: return "Bad Gateway";
- case StatusCode::ServiceUnavailable_503: return "Service Unavailable";
- case StatusCode::GatewayTimeout_504: return "Gateway Timeout";
- case StatusCode::HttpVersionNotSupported_505:
- return "HTTP Version Not Supported";
- case StatusCode::VariantAlsoNegotiates_506: return "Variant Also Negotiates";
- case StatusCode::InsufficientStorage_507: return "Insufficient Storage";
- case StatusCode::LoopDetected_508: return "Loop Detected";
- case StatusCode::NotExtended_510: return "Not Extended";
- case StatusCode::NetworkAuthenticationRequired_511:
- return "Network Authentication Required";
-
- default:
- case StatusCode::InternalServerError_500: return "Internal Server Error";
- }
-}
-
-inline std::string get_bearer_token_auth(const Request &req) {
- if (req.has_header("Authorization")) {
- static std::string BearerHeaderPrefix = "Bearer ";
- return req.get_header_value("Authorization")
- .substr(BearerHeaderPrefix.length());
- }
- return "";
-}
-
template <class Rep, class Period>
inline Server &
Server::set_read_timeout(const std::chrono::duration<Rep, Period> &duration) {
@@ -2001,7 +1720,6 @@ inline std::string to_string(const Error error) {
return "Unsupported HTTP multipart boundary characters";
case Error::Compression: return "Compression failed";
case Error::ConnectionTimeout: return "Connection timed out";
- case Error::ProxyConnection: return "Proxy connection failed";
case Error::Unknown: return "Unknown";
default: break;
}
@@ -2015,9 +1733,10 @@ inline std::ostream &operator<<(std::ostream &os, const Error &obj) {
return os;
}
-inline uint64_t Result::get_request_header_value_u64(const std::string &key,
- size_t id) const {
- return detail::get_header_value_u64(request_headers_, key, id, 0);
+template <typename T>
+inline T Result::get_request_header_value(const std::string &key,
+ size_t id) const {
+ return detail::get_header_value<T>(request_headers_, key, id, 0);
}
template <class Rep, class Period>
@@ -2071,7 +1790,7 @@ void hosted_at(const std::string &hostname, std::vector<std::string> &addrs);
std::string append_query_params(const std::string &path, const Params &params);
-std::pair<std::string, std::string> make_range_header(const Ranges &ranges);
+std::pair<std::string, std::string> make_range_header(Ranges ranges);
std::pair<std::string, std::string>
make_basic_authentication_header(const std::string &username,
@@ -2091,9 +1810,6 @@ std::string trim_copy(const std::string &s);
void split(const char *b, const char *e, char d,
std::function<void(const char *, const char *)> fn);
-void split(const char *b, const char *e, char d, size_t m,
- std::function<void(const char *, const char *)> fn);
-
bool process_client_socket(socket_t sock, time_t read_timeout_sec,
time_t read_timeout_usec, time_t write_timeout_sec,
time_t write_timeout_usec,
@@ -2128,7 +1844,7 @@ enum class EncodingType { None = 0, Gzip, Brotli };
EncodingType encoding_type(const Request &req, const Response &res);
-class BufferStream final : public Stream {
+class BufferStream : public Stream {
public:
BufferStream() = default;
~BufferStream() override = default;
@@ -2168,19 +1884,19 @@ public:
Callback callback) = 0;
};
-class nocompressor final : public compressor {
+class nocompressor : public compressor {
public:
- ~nocompressor() override = default;
+ virtual ~nocompressor() = default;
bool compress(const char *data, size_t data_length, bool /*last*/,
Callback callback) override;
};
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
-class gzip_compressor final : public compressor {
+class gzip_compressor : public compressor {
public:
gzip_compressor();
- ~gzip_compressor() override;
+ ~gzip_compressor();
bool compress(const char *data, size_t data_length, bool last,
Callback callback) override;
@@ -2190,10 +1906,10 @@ private:
z_stream strm_;
};
-class gzip_decompressor final : public decompressor {
+class gzip_decompressor : public decompressor {
public:
gzip_decompressor();
- ~gzip_decompressor() override;
+ ~gzip_decompressor();
bool is_valid() const override;
@@ -2207,7 +1923,7 @@ private:
#endif
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
-class brotli_compressor final : public compressor {
+class brotli_compressor : public compressor {
public:
brotli_compressor();
~brotli_compressor();
@@ -2219,7 +1935,7 @@ private:
BrotliEncoderState *state_ = nullptr;
};
-class brotli_decompressor final : public decompressor {
+class brotli_decompressor : public decompressor {
public:
brotli_decompressor();
~brotli_decompressor();
@@ -2256,29 +1972,6 @@ private:
std::string glowable_buffer_;
};
-class mmap {
-public:
- mmap(const char *path);
- ~mmap();
-
- bool open(const char *path);
- void close();
-
- bool is_open() const;
- size_t size() const;
- const char *data() const;
-
-private:
-#if defined(_WIN32)
- HANDLE hFile_;
- HANDLE hMapping_;
-#else
- int fd_;
-#endif
- size_t size_;
- void *addr_;
-};
-
} // namespace detail
// ----------------------------------------------------------------------------
@@ -2310,7 +2003,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,
val = 0;
for (; cnt; i++, cnt--) {
if (!s[i]) { return false; }
- auto v = 0;
+ int v = 0;
if (is_hex(s[i], v)) {
val = val * 16 + v;
} else {
@@ -2321,7 +2014,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,
}
inline std::string from_i_to_hex(size_t n) {
- static const auto charset = "0123456789abcdef";
+ const char *charset = "0123456789abcdef";
std::string ret;
do {
ret = charset[n & 15] + ret;
@@ -2332,7 +2025,7 @@ inline std::string from_i_to_hex(size_t n) {
inline size_t to_utf8(int code, char *buff) {
if (code < 0x0080) {
- buff[0] = static_cast<char>(code & 0x7F);
+ buff[0] = (code & 0x7F);
return 1;
} else if (code < 0x0800) {
buff[0] = static_cast<char>(0xC0 | ((code >> 6) & 0x1F));
@@ -2371,8 +2064,8 @@ inline std::string base64_encode(const std::string &in) {
std::string out;
out.reserve(in.size());
- auto val = 0;
- auto valb = -6;
+ int val = 0;
+ int valb = -6;
for (auto c : in) {
val = (val << 8) + static_cast<uint8_t>(c);
@@ -2419,11 +2112,6 @@ inline bool is_valid_path(const std::string &path) {
// Read component
auto beg = i;
while (i < path.size() && path[i] != '/') {
- if (path[i] == '\0') {
- return false;
- } else if (path[i] == '\\') {
- return false;
- }
i++;
}
@@ -2508,7 +2196,7 @@ inline std::string decode_url(const std::string &s,
for (size_t i = 0; i < s.size(); i++) {
if (s[i] == '%' && i + 1 < s.size()) {
if (s[i + 1] == 'u') {
- auto val = 0;
+ int val = 0;
if (from_hex_to_i(s, i + 2, 4, val)) {
// 4 digits Unicode codes
char buff[4];
@@ -2519,7 +2207,7 @@ inline std::string decode_url(const std::string &s,
result += s[i];
}
} else {
- auto val = 0;
+ int val = 0;
if (from_hex_to_i(s, i + 1, 2, val)) {
// 2 digits hex codes
result += static_cast<char>(val);
@@ -2572,30 +2260,16 @@ inline std::string trim_copy(const std::string &s) {
return s.substr(r.first, r.second - r.first);
}
-inline std::string trim_double_quotes_copy(const std::string &s) {
- if (s.length() >= 2 && s.front() == '"' && s.back() == '"') {
- return s.substr(1, s.size() - 2);
- }
- return s;
-}
-
inline void split(const char *b, const char *e, char d,
std::function<void(const char *, const char *)> fn) {
- return split(b, e, d, (std::numeric_limits<size_t>::max)(), std::move(fn));
-}
-
-inline void split(const char *b, const char *e, char d, size_t m,
- std::function<void(const char *, const char *)> fn) {
size_t i = 0;
size_t beg = 0;
- size_t count = 1;
while (e ? (b + i < e) : (b[i] != '\0')) {
- if (b[i] == d && count < m) {
+ if (b[i] == d) {
auto r = trim(b, e, beg, i);
if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }
beg = i + 1;
- count++;
}
i++;
}
@@ -2671,105 +2345,6 @@ inline void stream_line_reader::append(char c) {
}
}
-inline mmap::mmap(const char *path)
-#if defined(_WIN32)
- : hFile_(NULL), hMapping_(NULL)
-#else
- : fd_(-1)
-#endif
- ,
- size_(0), addr_(nullptr) {
- open(path);
-}
-
-inline mmap::~mmap() { close(); }
-
-inline bool mmap::open(const char *path) {
- close();
-
-#if defined(_WIN32)
- std::wstring wpath;
- for (size_t i = 0; i < strlen(path); i++) {
- wpath += path[i];
- }
-
- hFile_ = ::CreateFile2(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ,
- OPEN_EXISTING, NULL);
-
- if (hFile_ == INVALID_HANDLE_VALUE) { return false; }
-
- LARGE_INTEGER size{};
- if (!::GetFileSizeEx(hFile_, &size)) { return false; }
- size_ = static_cast<size_t>(size.QuadPart);
-
- hMapping_ =
- ::CreateFileMappingFromApp(hFile_, NULL, PAGE_READONLY, size_, NULL);
-
- if (hMapping_ == NULL) {
- close();
- return false;
- }
-
- addr_ = ::MapViewOfFileFromApp(hMapping_, FILE_MAP_READ, 0, 0);
-#else
- fd_ = ::open(path, O_RDONLY);
- if (fd_ == -1) { return false; }
-
- struct stat sb;
- if (fstat(fd_, &sb) == -1) {
- close();
- return false;
- }
- size_ = static_cast<size_t>(sb.st_size);
-
- addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0);
-#endif
-
- if (addr_ == nullptr) {
- close();
- return false;
- }
-
- return true;
-}
-
-inline bool mmap::is_open() const { return addr_ != nullptr; }
-
-inline size_t mmap::size() const { return size_; }
-
-inline const char *mmap::data() const {
- return static_cast<const char *>(addr_);
-}
-
-inline void mmap::close() {
-#if defined(_WIN32)
- if (addr_) {
- ::UnmapViewOfFile(addr_);
- addr_ = nullptr;
- }
-
- if (hMapping_) {
- ::CloseHandle(hMapping_);
- hMapping_ = NULL;
- }
-
- if (hFile_ != INVALID_HANDLE_VALUE) {
- ::CloseHandle(hFile_);
- hFile_ = INVALID_HANDLE_VALUE;
- }
-#else
- if (addr_ != nullptr) {
- munmap(addr_, size_);
- addr_ = nullptr;
- }
-
- if (fd_ != -1) {
- ::close(fd_);
- fd_ = -1;
- }
-#endif
- size_ = 0;
-}
inline int close_socket(socket_t sock) {
#ifdef _WIN32
return closesocket(sock);
@@ -2779,7 +2354,7 @@ inline int close_socket(socket_t sock) {
}
template <typename T> inline ssize_t handle_EINTR(T fn) {
- ssize_t res = 0;
+ ssize_t res = false;
while (true) {
res = fn();
if (res < 0 && errno == EINTR) { continue; }
@@ -2824,7 +2399,7 @@ inline ssize_t select_read(socket_t sock, time_t sec, time_t usec) {
return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });
#else
#ifndef _WIN32
- if (sock >= FD_SETSIZE) { return -1; }
+ if (sock >= FD_SETSIZE) { return 1; }
#endif
fd_set fds;
@@ -2852,7 +2427,7 @@ inline ssize_t select_write(socket_t sock, time_t sec, time_t usec) {
return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });
#else
#ifndef _WIN32
- if (sock >= FD_SETSIZE) { return -1; }
+ if (sock >= FD_SETSIZE) { return 1; }
#endif
fd_set fds;
@@ -2883,7 +2458,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec,
if (poll_res == 0) { return Error::ConnectionTimeout; }
if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) {
- auto error = 0;
+ int error = 0;
socklen_t len = sizeof(error);
auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,
reinterpret_cast<char *>(&error), &len);
@@ -2915,7 +2490,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec,
if (ret == 0) { return Error::ConnectionTimeout; }
if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) {
- auto error = 0;
+ int error = 0;
socklen_t len = sizeof(error);
auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,
reinterpret_cast<char *>(&error), &len);
@@ -2937,7 +2512,7 @@ inline bool is_socket_alive(socket_t sock) {
return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0;
}
-class SocketStream final : public Stream {
+class SocketStream : public Stream {
public:
SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,
time_t write_timeout_sec, time_t write_timeout_usec);
@@ -2962,11 +2537,11 @@ private:
size_t read_buff_off_ = 0;
size_t read_buff_content_size_ = 0;
- static const size_t read_buff_size_ = 1024l * 4;
+ static const size_t read_buff_size_ = 1024 * 4;
};
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
-class SSLSocketStream final : public Stream {
+class SSLSocketStream : public Stream {
public:
SSLSocketStream(socket_t sock, SSL *ssl, time_t read_timeout_sec,
time_t read_timeout_usec, time_t write_timeout_sec,
@@ -3091,7 +2666,7 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,
#ifndef _WIN32
if (hints.ai_family == AF_UNIX) {
const auto addrlen = host.length();
- if (addrlen > sizeof(sockaddr_un::sun_path)) { return INVALID_SOCKET; }
+ if (addrlen > sizeof(sockaddr_un::sun_path)) return INVALID_SOCKET;
auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol);
if (sock != INVALID_SOCKET) {
@@ -3160,27 +2735,17 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,
#endif
if (tcp_nodelay) {
- auto yes = 1;
-#ifdef _WIN32
- setsockopt(sock, IPPROTO_TCP, TCP_NODELAY,
- reinterpret_cast<const char *>(&yes), sizeof(yes));
-#else
- setsockopt(sock, IPPROTO_TCP, TCP_NODELAY,
- reinterpret_cast<const void *>(&yes), sizeof(yes));
-#endif
+ int yes = 1;
+ setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char *>(&yes),
+ sizeof(yes));
}
if (socket_options) { socket_options(sock); }
if (rp->ai_family == AF_INET6) {
- auto no = 0;
-#ifdef _WIN32
- setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY,
- reinterpret_cast<const char *>(&no), sizeof(no));
-#else
- setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY,
- reinterpret_cast<const void *>(&no), sizeof(no));
-#endif
+ int no = 0;
+ setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast<char *>(&no),
+ sizeof(no));
}
// bind or connect
@@ -3239,7 +2804,7 @@ inline bool bind_ip_address(socket_t sock, const std::string &host) {
return ret;
}
-#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__
+#if !defined _WIN32 && !defined ANDROID && !defined _AIX
#define USE_IF2IP
#endif
@@ -3295,7 +2860,7 @@ inline socket_t create_client_socket(
#ifdef USE_IF2IP
auto ip_from_if = if2ip(address_family, intf);
if (ip_from_if.empty()) { ip_from_if = intf; }
- if (!bind_ip_address(sock2, ip_from_if)) {
+ if (!bind_ip_address(sock2, ip_from_if.c_str())) {
error = Error::BindIPAddress;
return false;
}
@@ -3323,14 +2888,13 @@ inline socket_t create_client_socket(
#ifdef _WIN32
auto timeout = static_cast<uint32_t>(read_timeout_sec * 1000 +
read_timeout_usec / 1000);
- setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO,
- reinterpret_cast<const char *>(&timeout), sizeof(timeout));
+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,
+ sizeof(timeout));
#else
timeval tv;
tv.tv_sec = static_cast<long>(read_timeout_sec);
tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec);
- setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO,
- reinterpret_cast<const void *>(&tv), sizeof(tv));
+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));
#endif
}
{
@@ -3338,14 +2902,13 @@ inline socket_t create_client_socket(
#ifdef _WIN32
auto timeout = static_cast<uint32_t>(write_timeout_sec * 1000 +
write_timeout_usec / 1000);
- setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO,
- reinterpret_cast<const char *>(&timeout), sizeof(timeout));
+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,
+ sizeof(timeout));
#else
timeval tv;
tv.tv_sec = static_cast<long>(write_timeout_sec);
tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec);
- setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO,
- reinterpret_cast<const void *>(&tv), sizeof(tv));
+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));
#endif
}
@@ -3445,20 +3008,18 @@ inline constexpr unsigned int operator"" _t(const char *s, size_t l) {
} // namespace udl
-inline std::string
+inline const char *
find_content_type(const std::string &path,
- const std::map<std::string, std::string> &user_data,
- const std::string &default_content_type) {
+ const std::map<std::string, std::string> &user_data) {
auto ext = file_extension(path);
auto it = user_data.find(ext);
- if (it != user_data.end()) { return it->second; }
+ if (it != user_data.end()) { return it->second.c_str(); }
using udl::operator""_t;
switch (str2tag(ext)) {
- default: return default_content_type;
-
+ default: return nullptr;
case "css"_t: return "text/css";
case "csv"_t: return "text/csv";
case "htm"_t:
@@ -3511,6 +3072,76 @@ find_content_type(const std::string &path,
}
}
+inline const char *status_message(int status) {
+ switch (status) {
+ case 100: return "Continue";
+ case 101: return "Switching Protocol";
+ case 102: return "Processing";
+ case 103: return "Early Hints";
+ case 200: return "OK";
+ case 201: return "Created";
+ case 202: return "Accepted";
+ case 203: return "Non-Authoritative Information";
+ case 204: return "No Content";
+ case 205: return "Reset Content";
+ case 206: return "Partial Content";
+ case 207: return "Multi-Status";
+ case 208: return "Already Reported";
+ case 226: return "IM Used";
+ case 300: return "Multiple Choice";
+ case 301: return "Moved Permanently";
+ case 302: return "Found";
+ case 303: return "See Other";
+ case 304: return "Not Modified";
+ case 305: return "Use Proxy";
+ case 306: return "unused";
+ case 307: return "Temporary Redirect";
+ case 308: return "Permanent Redirect";
+ case 400: return "Bad Request";
+ case 401: return "Unauthorized";
+ case 402: return "Payment Required";
+ case 403: return "Forbidden";
+ case 404: return "Not Found";
+ case 405: return "Method Not Allowed";
+ case 406: return "Not Acceptable";
+ case 407: return "Proxy Authentication Required";
+ case 408: return "Request Timeout";
+ case 409: return "Conflict";
+ case 410: return "Gone";
+ case 411: return "Length Required";
+ case 412: return "Precondition Failed";
+ case 413: return "Payload Too Large";
+ case 414: return "URI Too Long";
+ case 415: return "Unsupported Media Type";
+ case 416: return "Range Not Satisfiable";
+ case 417: return "Expectation Failed";
+ case 418: return "I'm a teapot";
+ case 421: return "Misdirected Request";
+ case 422: return "Unprocessable Entity";
+ case 423: return "Locked";
+ case 424: return "Failed Dependency";
+ case 425: return "Too Early";
+ case 426: return "Upgrade Required";
+ case 428: return "Precondition Required";
+ case 429: return "Too Many Requests";
+ case 431: return "Request Header Fields Too Large";
+ case 451: return "Unavailable For Legal Reasons";
+ case 501: return "Not Implemented";
+ case 502: return "Bad Gateway";
+ case 503: return "Service Unavailable";
+ case 504: return "Gateway Timeout";
+ case 505: return "HTTP Version Not Supported";
+ case 506: return "Variant Also Negotiates";
+ case 507: return "Insufficient Storage";
+ case 508: return "Loop Detected";
+ case 510: return "Not Extended";
+ case 511: return "Network Authentication Required";
+
+ default:
+ case 500: return "Internal Server Error";
+ }
+}
+
inline bool can_compress_content_type(const std::string &content_type) {
using udl::operator""_t;
@@ -3587,7 +3218,7 @@ inline bool gzip_compressor::compress(const char *data, size_t data_length,
data += strm_.avail_in;
auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH;
- auto ret = Z_OK;
+ int ret = Z_OK;
std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
do {
@@ -3631,7 +3262,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,
Callback callback) {
assert(is_valid_);
- auto ret = Z_OK;
+ int ret = Z_OK;
do {
constexpr size_t max_avail_in =
@@ -3645,12 +3276,16 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,
data += strm_.avail_in;
std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
- while (strm_.avail_in > 0 && ret == Z_OK) {
+ while (strm_.avail_in > 0) {
strm_.avail_out = static_cast<uInt>(buff.size());
strm_.next_out = reinterpret_cast<Bytef *>(buff.data());
+ auto prev_avail_in = strm_.avail_in;
+
ret = inflate(&strm_, Z_NO_FLUSH);
+ if (prev_avail_in - strm_.avail_in == 0) { return false; }
+
assert(ret != Z_STREAM_ERROR);
switch (ret) {
case Z_NEED_DICT:
@@ -3663,7 +3298,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,
}
}
- if (ret != Z_OK && ret != Z_STREAM_END) { return false; }
+ if (ret != Z_OK && ret != Z_STREAM_END) return false;
} while (data_length > 0);
@@ -3732,7 +3367,7 @@ inline bool brotli_decompressor::decompress(const char *data,
return 0;
}
- auto next_in = reinterpret_cast<const uint8_t *>(data);
+ const uint8_t *next_in = (const uint8_t *)data;
size_t avail_in = data_length;
size_t total_out;
@@ -3802,9 +3437,6 @@ inline bool parse_header(const char *beg, const char *end, T fn) {
}
if (p < end) {
- auto key_len = key_end - beg;
- if (!key_len) { return false; }
-
auto key = std::string(beg, key_end);
auto val = compare_case_ignore(key, "Location")
? std::string(p, end)
@@ -3894,7 +3526,11 @@ inline bool read_content_without_length(Stream &strm,
uint64_t r = 0;
for (;;) {
auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ);
- if (n <= 0) { return true; }
+ if (n < 0) {
+ return false;
+ } else if (n == 0) {
+ return true;
+ }
if (!out(buf, static_cast<size_t>(n), r, 0)) { return false; }
r += static_cast<uint64_t>(n);
@@ -3930,7 +3566,7 @@ inline bool read_content_chunked(Stream &strm, T &x,
if (!line_reader.getline()) { return false; }
- if (strcmp(line_reader.ptr(), "\r\n") != 0) { return false; }
+ if (strcmp(line_reader.ptr(), "\r\n")) { return false; }
if (!line_reader.getline()) { return false; }
}
@@ -3940,7 +3576,7 @@ inline bool read_content_chunked(Stream &strm, T &x,
// Trailer
if (!line_reader.getline()) { return false; }
- while (strcmp(line_reader.ptr(), "\r\n") != 0) {
+ while (strcmp(line_reader.ptr(), "\r\n")) {
if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }
// Exclude line terminator
@@ -3959,8 +3595,8 @@ inline bool read_content_chunked(Stream &strm, T &x,
}
inline bool is_chunked_transfer_encoding(const Headers &headers) {
- return compare_case_ignore(
- get_header_value(headers, "Transfer-Encoding", 0, ""), "chunked");
+ return !strcasecmp(get_header_value(headers, "Transfer-Encoding", 0, ""),
+ "chunked");
}
template <typename T, typename U>
@@ -3975,14 +3611,14 @@ bool prepare_content_receiver(T &x, int &status,
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
decompressor = detail::make_unique<gzip_decompressor>();
#else
- status = StatusCode::UnsupportedMediaType_415;
+ status = 415;
return false;
#endif
} else if (encoding.find("br") != std::string::npos) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
decompressor = detail::make_unique<brotli_decompressor>();
#else
- status = StatusCode::UnsupportedMediaType_415;
+ status = 415;
return false;
#endif
}
@@ -3998,7 +3634,7 @@ bool prepare_content_receiver(T &x, int &status,
};
return callback(std::move(out));
} else {
- status = StatusCode::InternalServerError_500;
+ status = 500;
return false;
}
}
@@ -4026,7 +3662,7 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,
} else if (!has_header(x.headers, "Content-Length")) {
ret = read_content_without_length(strm, out);
} else {
- auto len = get_header_value_u64(x.headers, "Content-Length", 0, 0);
+ auto len = get_header_value<uint64_t>(x.headers, "Content-Length");
if (len > payload_max_length) {
exceed_payload_max_length = true;
skip_content_with_length(strm, len);
@@ -4036,10 +3672,7 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,
}
}
- if (!ret) {
- status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413
- : StatusCode::BadRequest_400;
- }
+ if (!ret) { status = exceed_payload_max_length ? 413 : 400; }
return ret;
});
} // namespace detail
@@ -4087,8 +3720,6 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider,
return ok;
};
- data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };
-
while (offset < end_offset && !is_shutting_down()) {
if (!strm.is_writable()) {
error = Error::Write;
@@ -4133,8 +3764,6 @@ write_content_without_length(Stream &strm,
return ok;
};
- data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };
-
data_sink.done = [&](void) { data_available = false; };
while (data_available && !is_shutting_down()) {
@@ -4185,8 +3814,6 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider,
return ok;
};
- data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };
-
auto done_with_trailer = [&](const Headers *trailer) {
if (!ok) { return; }
@@ -4271,8 +3898,7 @@ inline bool redirect(T &cli, Request &req, Response &res,
new_req.path = path;
new_req.redirect_count_ -= 1;
- if (res.status == StatusCode::SeeOther_303 &&
- (req.method != "GET" && req.method != "HEAD")) {
+ if (res.status == 303 && (req.method != "GET" && req.method != "HEAD")) {
new_req.method = "GET";
new_req.body.clear();
new_req.headers.clear();
@@ -4284,8 +3910,7 @@ inline bool redirect(T &cli, Request &req, Response &res,
if (ret) {
req = new_req;
res = new_res;
-
- if (res.location.empty()) { res.location = location; }
+ res.location = location;
}
return ret;
}
@@ -4332,34 +3957,14 @@ inline bool parse_multipart_boundary(const std::string &content_type,
if (pos == std::string::npos) { return false; }
auto end = content_type.find(';', pos);
auto beg = pos + strlen(boundary_keyword);
- boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg));
+ boundary = content_type.substr(beg, end - beg);
+ if (boundary.length() >= 2 && boundary.front() == '"' &&
+ boundary.back() == '"') {
+ boundary = boundary.substr(1, boundary.size() - 2);
+ }
return !boundary.empty();
}
-inline void parse_disposition_params(const std::string &s, Params &params) {
- std::set<std::string> cache;
- split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) {
- std::string kv(b, e);
- if (cache.find(kv) != cache.end()) { return; }
- cache.insert(kv);
-
- std::string key;
- std::string val;
- split(b, e, '=', [&](const char *b2, const char *e2) {
- if (key.empty()) {
- key.assign(b2, e2);
- } else {
- val.assign(b2, e2);
- }
- });
-
- if (!key.empty()) {
- params.emplace(trim_double_quotes_copy((key)),
- trim_double_quotes_copy((val)));
- }
- });
-}
-
#ifdef CPPHTTPLIB_NO_EXCEPTIONS
inline bool parse_range_header(const std::string &s, Ranges &ranges) {
#else
@@ -4370,9 +3975,9 @@ inline bool parse_range_header(const std::string &s, Ranges &ranges) try {
if (std::regex_match(s, m, re_first_range)) {
auto pos = static_cast<size_t>(m.position(1));
auto len = static_cast<size_t>(m.length(1));
- auto all_valid_ranges = true;
+ bool all_valid_ranges = true;
split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {
- if (!all_valid_ranges) { return; }
+ if (!all_valid_ranges) return;
static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))");
std::cmatch cm;
if (std::regex_match(b, e, cm, re_another_range)) {
@@ -4417,6 +4022,11 @@ public:
bool parse(const char *buf, size_t n, const ContentReceiver &content_callback,
const MultipartContentHeader &header_callback) {
+ // TODO: support 'filename*'
+ static const std::regex re_content_disposition(
+ R"~(^Content-Disposition:\s*form-data;\s*name="(.*?)"(?:;\s*filename="(.*?)")?(?:;\s*filename\*=\S+)?\s*$)~",
+ std::regex_constants::icase);
+
buf_append(buf, n);
while (buf_size() > 0) {
@@ -4449,54 +4059,18 @@ public:
break;
}
+ static const std::string header_name = "content-type:";
const auto header = buf_head(pos);
-
- if (!parse_header(header.data(), header.data() + header.size(),
- [&](std::string &&, std::string &&) {})) {
- is_valid_ = false;
- return false;
- }
-
- static const std::string header_content_type = "Content-Type:";
-
- if (start_with_case_ignore(header, header_content_type)) {
- file_.content_type =
- trim_copy(header.substr(header_content_type.size()));
+ if (start_with_case_ignore(header, header_name)) {
+ file_.content_type = trim_copy(header.substr(header_name.size()));
} else {
- static const std::regex re_content_disposition(
- R"~(^Content-Disposition:\s*form-data;\s*(.*)$)~",
- std::regex_constants::icase);
-
std::smatch m;
if (std::regex_match(header, m, re_content_disposition)) {
- Params params;
- parse_disposition_params(m[1], params);
-
- auto it = params.find("name");
- if (it != params.end()) {
- file_.name = it->second;
- } else {
- is_valid_ = false;
- return false;
- }
-
- it = params.find("filename");
- if (it != params.end()) { file_.filename = it->second; }
-
- it = params.find("filename*");
- if (it != params.end()) {
- // Only allow UTF-8 enconnding...
- static const std::regex re_rfc5987_encoding(
- R"~(^UTF-8''(.+?)$)~", std::regex_constants::icase);
-
- std::smatch m2;
- if (std::regex_match(it->second, m2, re_rfc5987_encoding)) {
- file_.filename = decode_url(m2[1], false); // override...
- } else {
- is_valid_ = false;
- return false;
- }
- }
+ file_.name = m[1];
+ file_.filename = m[2];
+ } else {
+ is_valid_ = false;
+ return false;
}
}
buf_erase(pos + crlf_.size());
@@ -4534,9 +4108,9 @@ public:
buf_erase(crlf_.size());
state_ = 1;
} else {
- if (dash_.size() > buf_size()) { return true; }
- if (buf_start_with(dash_)) {
- buf_erase(dash_.size());
+ if (dash_crlf_.size() > buf_size()) { return true; }
+ if (buf_start_with(dash_crlf_)) {
+ buf_erase(dash_crlf_.size());
is_valid_ = true;
buf_erase(buf_size()); // Remove epilogue
} else {
@@ -4569,6 +4143,7 @@ private:
const std::string dash_ = "--";
const std::string crlf_ = "\r\n";
+ const std::string dash_crlf_ = "--\r\n";
std::string boundary_;
std::string dash_boundary_crlf_;
std::string crlf_dash_boundary_;
@@ -4655,30 +4230,26 @@ inline std::string to_lower(const char *beg, const char *end) {
return out;
}
-inline std::string random_string(size_t length) {
+inline std::string make_multipart_data_boundary() {
static const char data[] =
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
// std::random_device might actually be deterministic on some
// platforms, but due to lack of support in the c++ standard library,
// doing better requires either some ugly hacks or breaking portability.
- static std::random_device seed_gen;
+ std::random_device seed_gen;
// Request 128 bits of entropy for initialization
- static std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(),
- seed_gen()};
+ std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()};
+ std::mt19937 engine(seed_sequence);
- static std::mt19937 engine(seed_sequence);
+ std::string result = "--cpp-httplib-multipart-data-";
- std::string result;
- for (size_t i = 0; i < length; i++) {
+ for (auto i = 0; i < 16; i++) {
result += data[engine() % (sizeof(data) - 1)];
}
- return result;
-}
-inline std::string make_multipart_data_boundary() {
- return "--cpp-httplib-multipart-data-" + detail::random_string(16);
+ return result;
}
inline bool is_multipart_boundary_chars_valid(const std::string &boundary) {
@@ -4733,97 +4304,48 @@ serialize_multipart_formdata(const MultipartFormDataItems &items,
body += item.content + serialize_multipart_formdata_item_end();
}
- if (finish) { body += serialize_multipart_formdata_finish(boundary); }
+ if (finish) body += serialize_multipart_formdata_finish(boundary);
return body;
}
-inline bool range_error(Request &req, Response &res) {
- if (!req.ranges.empty() && 200 <= res.status && res.status < 300) {
- ssize_t contant_len = static_cast<ssize_t>(
- res.content_length_ ? res.content_length_ : res.body.size());
-
- ssize_t prev_first_pos = -1;
- ssize_t prev_last_pos = -1;
- size_t overwrapping_count = 0;
-
- // NOTE: The following Range check is based on '14.2. Range' in RFC 9110
- // 'HTTP Semantics' to avoid potential denial-of-service attacks.
- // https://www.rfc-editor.org/rfc/rfc9110#section-14.2
-
- // Too many ranges
- if (req.ranges.size() > CPPHTTPLIB_RANGE_MAX_COUNT) { return true; }
-
- for (auto &r : req.ranges) {
- auto &first_pos = r.first;
- auto &last_pos = r.second;
-
- if (first_pos == -1 && last_pos == -1) {
- first_pos = 0;
- last_pos = contant_len;
- }
-
- if (first_pos == -1) {
- first_pos = contant_len - last_pos;
- last_pos = contant_len - 1;
- }
-
- if (last_pos == -1) { last_pos = contant_len - 1; }
-
- // Range must be within content length
- if (!(0 <= first_pos && first_pos <= last_pos &&
- last_pos <= contant_len - 1)) {
- return true;
- }
-
- // Ranges must be in ascending order
- if (first_pos <= prev_first_pos) { return true; }
-
- // Request must not have more than two overlapping ranges
- if (first_pos <= prev_last_pos) {
- overwrapping_count++;
- if (overwrapping_count > 2) { return true; }
- }
+inline std::pair<size_t, size_t>
+get_range_offset_and_length(const Request &req, size_t content_length,
+ size_t index) {
+ auto r = req.ranges[index];
- prev_first_pos = (std::max)(prev_first_pos, first_pos);
- prev_last_pos = (std::max)(prev_last_pos, last_pos);
- }
+ if (r.first == -1 && r.second == -1) {
+ return std::make_pair(0, content_length);
}
- return false;
-}
+ auto slen = static_cast<ssize_t>(content_length);
-inline std::pair<size_t, size_t>
-get_range_offset_and_length(Range r, size_t content_length) {
- (void)(content_length); // patch to get rid of "unused parameter" on release build
- assert(r.first != -1 && r.second != -1);
- assert(0 <= r.first && r.first < static_cast<ssize_t>(content_length));
- assert(r.first <= r.second &&
- r.second < static_cast<ssize_t>(content_length));
+ if (r.first == -1) {
+ r.first = (std::max)(static_cast<ssize_t>(0), slen - r.second);
+ r.second = slen - 1;
+ }
+ if (r.second == -1) { r.second = slen - 1; }
return std::make_pair(r.first, static_cast<size_t>(r.second - r.first) + 1);
}
-inline std::string make_content_range_header_field(
- const std::pair<size_t, size_t> &offset_and_length, size_t content_length) {
- auto st = offset_and_length.first;
- auto ed = st + offset_and_length.second - 1;
-
+inline std::string make_content_range_header_field(size_t offset, size_t length,
+ size_t content_length) {
std::string field = "bytes ";
- field += std::to_string(st);
+ field += std::to_string(offset);
field += "-";
- field += std::to_string(ed);
+ field += std::to_string(offset + length - 1);
field += "/";
field += std::to_string(content_length);
return field;
}
template <typename SToken, typename CToken, typename Content>
-bool process_multipart_ranges_data(const Request &req,
+bool process_multipart_ranges_data(const Request &req, Response &res,
const std::string &boundary,
const std::string &content_type,
- size_t content_length, SToken stoken,
- CToken ctoken, Content content) {
+ SToken stoken, CToken ctoken,
+ Content content) {
for (size_t i = 0; i < req.ranges.size(); i++) {
ctoken("--");
stoken(boundary);
@@ -4834,51 +4356,50 @@ bool process_multipart_ranges_data(const Request &req,
ctoken("\r\n");
}
- auto offset_and_length =
- get_range_offset_and_length(req.ranges[i], content_length);
+ auto offsets = get_range_offset_and_length(req, res.body.size(), i);
+ auto offset = offsets.first;
+ auto length = offsets.second;
ctoken("Content-Range: ");
- stoken(make_content_range_header_field(offset_and_length, content_length));
+ stoken(make_content_range_header_field(offset, length, res.body.size()));
ctoken("\r\n");
ctoken("\r\n");
-
- if (!content(offset_and_length.first, offset_and_length.second)) {
- return false;
- }
+ if (!content(offset, length)) { return false; }
ctoken("\r\n");
}
ctoken("--");
stoken(boundary);
- ctoken("--");
+ ctoken("--\r\n");
return true;
}
-inline void make_multipart_ranges_data(const Request &req, Response &res,
+inline bool make_multipart_ranges_data(const Request &req, Response &res,
const std::string &boundary,
const std::string &content_type,
- size_t content_length,
std::string &data) {
- process_multipart_ranges_data(
- req, boundary, content_type, content_length,
+ return process_multipart_ranges_data(
+ req, res, boundary, content_type,
[&](const std::string &token) { data += token; },
[&](const std::string &token) { data += token; },
[&](size_t offset, size_t length) {
- assert(offset + length <= content_length);
- data += res.body.substr(offset, length);
- return true;
+ if (offset < res.body.size()) {
+ data += res.body.substr(offset, length);
+ return true;
+ }
+ return false;
});
}
-inline size_t get_multipart_ranges_data_length(const Request &req,
- const std::string &boundary,
- const std::string &content_type,
- size_t content_length) {
+inline size_t
+get_multipart_ranges_data_length(const Request &req, Response &res,
+ const std::string &boundary,
+ const std::string &content_type) {
size_t data_length = 0;
process_multipart_ranges_data(
- req, boundary, content_type, content_length,
+ req, res, boundary, content_type,
[&](const std::string &token) { data_length += token.size(); },
[&](const std::string &token) { data_length += token.size(); },
[&](size_t /*offset*/, size_t length) {
@@ -4890,13 +4411,13 @@ inline size_t get_multipart_ranges_data_length(const Request &req,
}
template <typename T>
-inline bool
-write_multipart_ranges_data(Stream &strm, const Request &req, Response &res,
- const std::string &boundary,
- const std::string &content_type,
- size_t content_length, const T &is_shutting_down) {
+inline bool write_multipart_ranges_data(Stream &strm, const Request &req,
+ Response &res,
+ const std::string &boundary,
+ const std::string &content_type,
+ const T &is_shutting_down) {
return process_multipart_ranges_data(
- req, boundary, content_type, content_length,
+ req, res, boundary, content_type,
[&](const std::string &token) { strm.write(token); },
[&](const std::string &token) { strm.write(token); },
[&](size_t offset, size_t length) {
@@ -4905,6 +4426,18 @@ write_multipart_ranges_data(Stream &strm, const Request &req, Response &res,
});
}
+inline std::pair<size_t, size_t>
+get_range_offset_and_length(const Request &req, const Response &res,
+ size_t index) {
+ auto r = req.ranges[index];
+
+ if (r.second == -1) {
+ r.second = static_cast<ssize_t>(res.content_length_) - 1;
+ }
+
+ return std::make_pair(r.first, r.second - r.first + 1);
+}
+
inline bool expect_content(const Request &req) {
if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" ||
req.method == "PRI" || req.method == "DELETE") {
@@ -4938,7 +4471,7 @@ inline std::string message_digest(const std::string &s, const EVP_MD *algo) {
std::stringstream ss;
for (auto i = 0u; i < hash_length; ++i) {
ss << std::hex << std::setw(2) << std::setfill('0')
- << static_cast<unsigned int>(hash[i]);
+ << (unsigned int)hash[i];
}
return ss.str();
@@ -5031,7 +4564,7 @@ inline bool retrieve_root_certs_from_keychain(CFObjectPtr<CFArrayRef> &certs) {
inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) {
auto result = false;
- for (auto i = 0; i < CFArrayGetCount(certs); ++i) {
+ for (int i = 0; i < CFArrayGetCount(certs); ++i) {
const auto cert = reinterpret_cast<const __SecCertificate *>(
CFArrayGetValueAtIndex(certs, i));
@@ -5174,7 +4707,7 @@ inline bool parse_www_authenticate(const Response &res,
s = s.substr(pos + 1);
auto beg = std::sregex_iterator(s.begin(), s.end(), re);
for (auto i = beg; i != std::sregex_iterator(); ++i) {
- const auto &m = *i;
+ auto m = *i;
auto key = s.substr(static_cast<size_t>(m.position(1)),
static_cast<size_t>(m.length(1)));
auto val = m.length(2) > 0
@@ -5191,6 +4724,20 @@ inline bool parse_www_authenticate(const Response &res,
return false;
}
+// https://stackoverflow.com/questions/440133/how-do-i-create-a-random-alpha-numeric-string-in-c/440240#answer-440240
+inline std::string random_string(size_t length) {
+ auto randchar = []() -> char {
+ const char charset[] = "0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz";
+ const size_t max_index = (sizeof(charset) - 1);
+ return charset[static_cast<size_t>(std::rand()) % max_index];
+ };
+ std::string str(length, 0);
+ std::generate_n(str.begin(), length, randchar);
+ return str;
+}
+
class ContentProviderAdapter {
public:
explicit ContentProviderAdapter(
@@ -5235,7 +4782,7 @@ inline void hosted_at(const std::string &hostname,
const auto &addr =
*reinterpret_cast<struct sockaddr_storage *>(rp->ai_addr);
std::string ip;
- auto dummy = -1;
+ int dummy = -1;
if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip,
dummy)) {
addrs.push_back(ip);
@@ -5255,11 +4802,10 @@ inline std::string append_query_params(const std::string &path,
}
// Header utilities
-inline std::pair<std::string, std::string>
-make_range_header(const Ranges &ranges) {
+inline std::pair<std::string, std::string> make_range_header(Ranges ranges) {
std::string field = "bytes=";
auto i = 0;
- for (const auto &r : ranges) {
+ for (auto r : ranges) {
if (i != 0) { field += ", "; }
if (r.first != -1) { field += std::to_string(r.first); }
field += '-';
@@ -5378,7 +4924,7 @@ inline void Response::set_redirect(const std::string &url, int stat) {
if (300 <= stat && stat < 400) {
this->status = stat;
} else {
- this->status = StatusCode::Found_302;
+ this->status = 302;
}
}
}
@@ -5397,22 +4943,13 @@ inline void Response::set_content(const std::string &s,
set_content(s.data(), s.size(), content_type);
}
-inline void Response::set_content(std::string &&s,
- const std::string &content_type) {
- body = std::move(s);
-
- auto rng = headers.equal_range("Content-Type");
- headers.erase(rng.first, rng.second);
- set_header("Content-Type", content_type);
-}
-
inline void Response::set_content_provider(
size_t in_length, const std::string &content_type, ContentProvider provider,
ContentProviderResourceReleaser resource_releaser) {
set_header("Content-Type", content_type);
content_length_ = in_length;
if (in_length > 0) { content_provider_ = std::move(provider); }
- content_provider_resource_releaser_ = std::move(resource_releaser);
+ content_provider_resource_releaser_ = resource_releaser;
is_chunked_content_provider_ = false;
}
@@ -5422,7 +4959,7 @@ inline void Response::set_content_provider(
set_header("Content-Type", content_type);
content_length_ = 0;
content_provider_ = detail::ContentProviderAdapter(std::move(provider));
- content_provider_resource_releaser_ = std::move(resource_releaser);
+ content_provider_resource_releaser_ = resource_releaser;
is_chunked_content_provider_ = false;
}
@@ -5432,7 +4969,7 @@ inline void Response::set_chunked_content_provider(
set_header("Content-Type", content_type);
content_length_ = 0;
content_provider_ = detail::ContentProviderAdapter(std::move(provider));
- content_provider_resource_releaser_ = std::move(resource_releaser);
+ content_provider_resource_releaser_ = resource_releaser;
is_chunked_content_provider_ = true;
}
@@ -5473,7 +5010,7 @@ inline SocketStream::SocketStream(socket_t sock, time_t read_timeout_sec,
write_timeout_sec_(write_timeout_sec),
write_timeout_usec_(write_timeout_usec), read_buff_(read_buff_size_, 0) {}
-inline SocketStream::~SocketStream() = default;
+inline SocketStream::~SocketStream() {}
inline bool SocketStream::is_readable() const {
return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;
@@ -5583,99 +5120,6 @@ inline socket_t BufferStream::socket() const { return 0; }
inline const std::string &BufferStream::get_buffer() const { return buffer; }
-inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) {
- // One past the last ending position of a path param substring
- std::size_t last_param_end = 0;
-
-#ifndef CPPHTTPLIB_NO_EXCEPTIONS
- // Needed to ensure that parameter names are unique during matcher
- // construction
- // If exceptions are disabled, only last duplicate path
- // parameter will be set
- std::unordered_set<std::string> param_name_set;
-#endif
-
- while (true) {
- const auto marker_pos = pattern.find(marker, last_param_end);
- if (marker_pos == std::string::npos) { break; }
-
- static_fragments_.push_back(
- pattern.substr(last_param_end, marker_pos - last_param_end));
-
- const auto param_name_start = marker_pos + 1;
-
- auto sep_pos = pattern.find(separator, param_name_start);
- if (sep_pos == std::string::npos) { sep_pos = pattern.length(); }
-
- auto param_name =
- pattern.substr(param_name_start, sep_pos - param_name_start);
-
-#ifndef CPPHTTPLIB_NO_EXCEPTIONS
- if (param_name_set.find(param_name) != param_name_set.cend()) {
- std::string msg = "Encountered path parameter '" + param_name +
- "' multiple times in route pattern '" + pattern + "'.";
- throw std::invalid_argument(msg);
- }
-#endif
-
- param_names_.push_back(std::move(param_name));
-
- last_param_end = sep_pos + 1;
- }
-
- if (last_param_end < pattern.length()) {
- static_fragments_.push_back(pattern.substr(last_param_end));
- }
-}
-
-inline bool PathParamsMatcher::match(Request &request) const {
- request.matches = std::smatch();
- request.path_params.clear();
- request.path_params.reserve(param_names_.size());
-
- // One past the position at which the path matched the pattern last time
- std::size_t starting_pos = 0;
- for (size_t i = 0; i < static_fragments_.size(); ++i) {
- const auto &fragment = static_fragments_[i];
-
- if (starting_pos + fragment.length() > request.path.length()) {
- return false;
- }
-
- // Avoid unnecessary allocation by using strncmp instead of substr +
- // comparison
- if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(),
- fragment.length()) != 0) {
- return false;
- }
-
- starting_pos += fragment.length();
-
- // Should only happen when we have a static fragment after a param
- // Example: '/users/:id/subscriptions'
- // The 'subscriptions' fragment here does not have a corresponding param
- if (i >= param_names_.size()) { continue; }
-
- auto sep_pos = request.path.find(separator, starting_pos);
- if (sep_pos == std::string::npos) { sep_pos = request.path.length(); }
-
- const auto &param_name = param_names_[i];
-
- request.path_params.emplace(
- param_name, request.path.substr(starting_pos, sep_pos - starting_pos));
-
- // Mark everythin up to '/' as matched
- starting_pos = sep_pos + 1;
- }
- // Returns false if the path is longer than the pattern
- return starting_pos >= request.path.length();
-}
-
-inline bool RegexMatcher::match(Request &request) const {
- request.path_params.clear();
- return std::regex_match(request.path, request.matches, regex_);
-}
-
} // namespace detail
// HTTP server implementation
@@ -5687,72 +5131,69 @@ inline Server::Server()
#endif
}
-inline Server::~Server() = default;
-
-inline std::unique_ptr<detail::MatcherBase>
-Server::make_matcher(const std::string &pattern) {
- if (pattern.find("/:") != std::string::npos) {
- return detail::make_unique<detail::PathParamsMatcher>(pattern);
- } else {
- return detail::make_unique<detail::RegexMatcher>(pattern);
- }
-}
+inline Server::~Server() {}
inline Server &Server::Get(const std::string &pattern, Handler handler) {
- get_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+ get_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Post(const std::string &pattern, Handler handler) {
- post_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+ post_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Post(const std::string &pattern,
HandlerWithContentReader handler) {
- post_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
- std::move(handler));
+ post_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Put(const std::string &pattern, Handler handler) {
- put_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+ put_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Put(const std::string &pattern,
HandlerWithContentReader handler) {
- put_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
- std::move(handler));
+ put_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Patch(const std::string &pattern, Handler handler) {
- patch_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+ patch_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Patch(const std::string &pattern,
HandlerWithContentReader handler) {
- patch_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
- std::move(handler));
+ patch_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Delete(const std::string &pattern, Handler handler) {
- delete_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+ delete_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Delete(const std::string &pattern,
HandlerWithContentReader handler) {
- delete_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
- std::move(handler));
+ delete_handlers_for_content_reader_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
inline Server &Server::Options(const std::string &pattern, Handler handler) {
- options_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+ options_handlers_.push_back(
+ std::make_pair(std::regex(pattern), std::move(handler)));
return *this;
}
@@ -5790,11 +5231,6 @@ Server::set_file_extension_and_mimetype_mapping(const std::string &ext,
return *this;
}
-inline Server &Server::set_default_file_mimetype(const std::string &mime) {
- default_file_mimetype_ = mime;
- return *this;
-}
-
inline Server &Server::set_file_request_handler(Handler handler) {
file_request_handler_ = std::move(handler);
return *this;
@@ -5836,6 +5272,7 @@ inline Server &Server::set_logger(Logger logger) {
inline Server &
Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) {
expect_100_continue_handler_ = std::move(handler);
+
return *this;
}
@@ -5859,12 +5296,6 @@ inline Server &Server::set_default_headers(Headers headers) {
return *this;
}
-inline Server &Server::set_header_writer(
- std::function<ssize_t(Stream &, Headers &)> const &writer) {
- header_writer_ = writer;
- return *this;
-}
-
inline Server &Server::set_keep_alive_max_count(size_t count) {
keep_alive_max_count_ = count;
return *this;
@@ -5900,7 +5331,8 @@ inline Server &Server::set_payload_max_length(size_t length) {
inline bool Server::bind_to_port(const std::string &host, int port,
int socket_flags) {
- return bind_internal(host, port, socket_flags) >= 0;
+ if (bind_internal(host, port, socket_flags) < 0) return false;
+ return true;
}
inline int Server::bind_to_any_port(const std::string &host, int socket_flags) {
return bind_internal(host, 0, socket_flags);
@@ -5934,7 +5366,7 @@ inline void Server::stop() {
}
}
-inline bool Server::parse_request_line(const char *s, Request &req) const {
+inline bool Server::parse_request_line(const char *s, Request &req) {
auto len = strlen(s);
if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') { return false; }
len -= 2;
@@ -5975,7 +5407,7 @@ inline bool Server::parse_request_line(const char *s, Request &req) const {
size_t count = 0;
detail::split(req.target.data(), req.target.data() + req.target.size(), '?',
- 2, [&](const char *b, const char *e) {
+ [&](const char *b, const char *e) {
switch (count) {
case 0:
req.path = detail::decode_url(std::string(b, e), false);
@@ -5998,10 +5430,7 @@ inline bool Server::parse_request_line(const char *s, Request &req) const {
}
inline bool Server::write_response(Stream &strm, bool close_connection,
- Request &req, Response &res) {
- // NOTE: `req.ranges` should be empty, otherwise it will be applied
- // incorrectly to the error content.
- req.ranges.clear();
+ const Request &req, Response &res) {
return write_response_core(strm, close_connection, req, res, false);
}
@@ -6057,11 +5486,11 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection,
detail::BufferStream bstrm;
if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status,
- status_message(res.status))) {
+ detail::status_message(res.status))) {
return false;
}
- if (!header_writer_(bstrm, res.headers)) { return false; }
+ if (!detail::write_headers(bstrm, res.headers)) { return false; }
// Flush buffer
auto &data = bstrm.get_buffer();
@@ -6079,6 +5508,7 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection,
if (write_content_with_provider(strm, req, res, boundary, content_type)) {
res.content_provider_success_ = true;
} else {
+ res.content_provider_success_ = false;
ret = false;
}
}
@@ -6103,16 +5533,15 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
return detail::write_content(strm, res.content_provider_, 0,
res.content_length_, is_shutting_down);
} else if (req.ranges.size() == 1) {
- auto offset_and_length = detail::get_range_offset_and_length(
- req.ranges[0], res.content_length_);
-
- return detail::write_content(strm, res.content_provider_,
- offset_and_length.first,
- offset_and_length.second, is_shutting_down);
+ auto offsets =
+ detail::get_range_offset_and_length(req, res.content_length_, 0);
+ auto offset = offsets.first;
+ auto length = offsets.second;
+ return detail::write_content(strm, res.content_provider_, offset, length,
+ is_shutting_down);
} else {
return detail::write_multipart_ranges_data(
- strm, req, res, boundary, content_type, res.content_length_,
- is_shutting_down);
+ strm, req, res, boundary, content_type, is_shutting_down);
}
} else {
if (res.is_chunked_content_provider_) {
@@ -6169,7 +5598,7 @@ inline bool Server::read_content(Stream &strm, Request &req, Response &res) {
const auto &content_type = req.get_header_value("Content-Type");
if (!content_type.find("application/x-www-form-urlencoded")) {
if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) {
- res.status = StatusCode::PayloadTooLarge_413; // NOTE: should be 414?
+ res.status = 413; // NOTE: should be 414?
return false;
}
detail::parse_query_text(req.body, req.params);
@@ -6188,11 +5617,10 @@ inline bool Server::read_content_with_content_receiver(
std::move(multipart_receiver));
}
-inline bool
-Server::read_content_core(Stream &strm, Request &req, Response &res,
- ContentReceiver receiver,
- MultipartContentHeader multipart_header,
- ContentReceiver multipart_receiver) const {
+inline bool Server::read_content_core(Stream &strm, Request &req, Response &res,
+ ContentReceiver receiver,
+ MultipartContentHeader multipart_header,
+ ContentReceiver multipart_receiver) {
detail::MultipartFormDataParser multipart_form_data_parser;
ContentReceiverWithProgress out;
@@ -6200,7 +5628,7 @@ Server::read_content_core(Stream &strm, Request &req, Response &res,
const auto &content_type = req.get_header_value("Content-Type");
std::string boundary;
if (!detail::parse_multipart_boundary(content_type, boundary)) {
- res.status = StatusCode::BadRequest_400;
+ res.status = 400;
return false;
}
@@ -6236,7 +5664,7 @@ Server::read_content_core(Stream &strm, Request &req, Response &res,
if (req.is_multipart_form_data()) {
if (!multipart_form_data_parser.is_valid()) {
- res.status = StatusCode::BadRequest_400;
+ res.status = 400;
return false;
}
}
@@ -6255,26 +5683,17 @@ inline bool Server::handle_file_request(const Request &req, Response &res,
if (path.back() == '/') { path += "index.html"; }
if (detail::is_file(path)) {
+ detail::read_file(path, res.body);
+ auto type =
+ detail::find_content_type(path, file_extension_and_mimetype_map_);
+ if (type) { res.set_header("Content-Type", type); }
for (const auto &kv : entry.headers) {
- res.set_header(kv.first, kv.second);
+ res.set_header(kv.first.c_str(), kv.second);
}
-
- auto mm = std::make_shared<detail::mmap>(path.c_str());
- if (!mm->is_open()) { return false; }
-
- res.set_content_provider(
- mm->size(),
- detail::find_content_type(path, file_extension_and_mimetype_map_,
- default_file_mimetype_),
- [mm](size_t offset, size_t length, DataSink &sink) -> bool {
- sink.write(mm->data() + offset, length);
- return true;
- });
-
+ res.status = req.has_header("Range") ? 206 : 200;
if (!head && file_request_handler_) {
file_request_handler_(req, res);
}
-
return true;
}
}
@@ -6370,14 +5789,13 @@ inline bool Server::listen_internal() {
#ifdef _WIN32
auto timeout = static_cast<uint32_t>(read_timeout_sec_ * 1000 +
read_timeout_usec_ / 1000);
- setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO,
- reinterpret_cast<const char *>(&timeout), sizeof(timeout));
+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,
+ sizeof(timeout));
#else
timeval tv;
tv.tv_sec = static_cast<long>(read_timeout_sec_);
tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec_);
- setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO,
- reinterpret_cast<const void *>(&tv), sizeof(tv));
+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));
#endif
}
{
@@ -6385,22 +5803,17 @@ inline bool Server::listen_internal() {
#ifdef _WIN32
auto timeout = static_cast<uint32_t>(write_timeout_sec_ * 1000 +
write_timeout_usec_ / 1000);
- setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO,
- reinterpret_cast<const char *>(&timeout), sizeof(timeout));
+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,
+ sizeof(timeout));
#else
timeval tv;
tv.tv_sec = static_cast<long>(write_timeout_sec_);
tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec_);
- setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO,
- reinterpret_cast<const void *>(&tv), sizeof(tv));
+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));
#endif
}
- if (!task_queue->enqueue(
- [this, sock]() { process_and_close_socket(sock); })) {
- detail::shutdown_socket(sock);
- detail::close_socket(sock);
- }
+ task_queue->enqueue([this, sock]() { process_and_close_socket(sock); });
}
task_queue->shutdown();
@@ -6416,7 +5829,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) {
}
// File handler
- auto is_head_request = req.method == "HEAD";
+ bool is_head_request = req.method == "HEAD";
if ((req.method == "GET" || is_head_request) &&
handle_file_request(req, res, is_head_request)) {
return true;
@@ -6482,17 +5895,17 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) {
return dispatch_request(req, res, patch_handlers_);
}
- res.status = StatusCode::BadRequest_400;
+ res.status = 400;
return false;
}
inline bool Server::dispatch_request(Request &req, Response &res,
- const Handlers &handlers) const {
+ const Handlers &handlers) {
for (const auto &x : handlers) {
- const auto &matcher = x.first;
+ const auto &pattern = x.first;
const auto &handler = x.second;
- if (matcher->match(req)) {
+ if (std::regex_match(req.path, req.matches, pattern)) {
handler(req, res);
return true;
}
@@ -6502,18 +5915,18 @@ inline bool Server::dispatch_request(Request &req, Response &res,
inline void Server::apply_ranges(const Request &req, Response &res,
std::string &content_type,
- std::string &boundary) const {
+ std::string &boundary) {
if (req.ranges.size() > 1) {
+ boundary = detail::make_multipart_data_boundary();
+
auto it = res.headers.find("Content-Type");
if (it != res.headers.end()) {
content_type = it->second;
res.headers.erase(it);
}
- boundary = detail::make_multipart_data_boundary();
-
- res.set_header("Content-Type",
- "multipart/byteranges; boundary=" + boundary);
+ res.headers.emplace("Content-Type",
+ "multipart/byteranges; boundary=" + boundary);
}
auto type = detail::encoding_type(req, res);
@@ -6524,17 +5937,16 @@ inline void Server::apply_ranges(const Request &req, Response &res,
if (req.ranges.empty()) {
length = res.content_length_;
} else if (req.ranges.size() == 1) {
- auto offset_and_length = detail::get_range_offset_and_length(
- req.ranges[0], res.content_length_);
-
- length = offset_and_length.second;
-
+ auto offsets =
+ detail::get_range_offset_and_length(req, res.content_length_, 0);
+ auto offset = offsets.first;
+ length = offsets.second;
auto content_range = detail::make_content_range_header_field(
- offset_and_length, res.content_length_);
+ offset, length, res.content_length_);
res.set_header("Content-Range", content_range);
} else {
- length = detail::get_multipart_ranges_data_length(
- req, boundary, content_type, res.content_length_);
+ length = detail::get_multipart_ranges_data_length(req, res, boundary,
+ content_type);
}
res.set_header("Content-Length", std::to_string(length));
} else {
@@ -6553,22 +5965,28 @@ inline void Server::apply_ranges(const Request &req, Response &res,
if (req.ranges.empty()) {
;
} else if (req.ranges.size() == 1) {
- auto offset_and_length =
- detail::get_range_offset_and_length(req.ranges[0], res.body.size());
- auto offset = offset_and_length.first;
- auto length = offset_and_length.second;
-
+ auto offsets =
+ detail::get_range_offset_and_length(req, res.body.size(), 0);
+ auto offset = offsets.first;
+ auto length = offsets.second;
auto content_range = detail::make_content_range_header_field(
- offset_and_length, res.body.size());
+ offset, length, res.body.size());
res.set_header("Content-Range", content_range);
-
- assert(offset + length <= res.body.size());
- res.body = res.body.substr(offset, length);
+ if (offset < res.body.size()) {
+ res.body = res.body.substr(offset, length);
+ } else {
+ res.body.clear();
+ res.status = 416;
+ }
} else {
std::string data;
- detail::make_multipart_ranges_data(req, res, boundary, content_type,
- res.body.size(), data);
- res.body.swap(data);
+ if (detail::make_multipart_ranges_data(req, res, boundary, content_type,
+ data)) {
+ res.body.swap(data);
+ } else {
+ res.body.clear();
+ res.status = 416;
+ }
}
if (type != detail::EncodingType::None) {
@@ -6607,12 +6025,12 @@ inline void Server::apply_ranges(const Request &req, Response &res,
inline bool Server::dispatch_request_for_content_reader(
Request &req, Response &res, ContentReader content_reader,
- const HandlersForContentReader &handlers) const {
+ const HandlersForContentReader &handlers) {
for (const auto &x : handlers) {
- const auto &matcher = x.first;
+ const auto &pattern = x.first;
const auto &handler = x.second;
- if (matcher->match(req)) {
+ if (std::regex_match(req.path, req.matches, pattern)) {
handler(req, res, content_reader);
return true;
}
@@ -6632,10 +6050,15 @@ Server::process_request(Stream &strm, bool close_connection,
if (!line_reader.getline()) { return false; }
Request req;
-
Response res;
+
res.version = "HTTP/1.1";
- res.headers = default_headers_;
+
+ for (const auto &header : default_headers_) {
+ if (res.headers.find(header.first) == res.headers.end()) {
+ res.headers.insert(header);
+ }
+ }
#ifdef _WIN32
// TODO: Increase FD_SETSIZE statically (libzmq), dynamically (MySQL).
@@ -6645,7 +6068,7 @@ Server::process_request(Stream &strm, bool close_connection,
if (strm.socket() >= FD_SETSIZE) {
Headers dummy;
detail::read_headers(strm, dummy);
- res.status = StatusCode::InternalServerError_500;
+ res.status = 500;
return write_response(strm, close_connection, req, res);
}
#endif
@@ -6655,14 +6078,14 @@ Server::process_request(Stream &strm, bool close_connection,
if (line_reader.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) {
Headers dummy;
detail::read_headers(strm, dummy);
- res.status = StatusCode::UriTooLong_414;
+ res.status = 414;
return write_response(strm, close_connection, req, res);
}
// Request line and headers
if (!parse_request_line(line_reader.ptr(), req) ||
!detail::read_headers(strm, req.headers)) {
- res.status = StatusCode::BadRequest_400;
+ res.status = 400;
return write_response(strm, close_connection, req, res);
}
@@ -6686,7 +6109,7 @@ Server::process_request(Stream &strm, bool close_connection,
if (req.has_header("Range")) {
const auto &range_header_value = req.get_header_value("Range");
if (!detail::parse_range_header(range_header_value, req.ranges)) {
- res.status = StatusCode::RangeNotSatisfiable_416;
+ res.status = 416;
return write_response(strm, close_connection, req, res);
}
}
@@ -6694,22 +6117,22 @@ Server::process_request(Stream &strm, bool close_connection,
if (setup_request) { setup_request(req); }
if (req.get_header_value("Expect") == "100-continue") {
- int status = StatusCode::Continue_100;
+ auto status = 100;
if (expect_100_continue_handler_) {
status = expect_100_continue_handler_(req, res);
}
switch (status) {
- case StatusCode::Continue_100:
- case StatusCode::ExpectationFailed_417:
+ case 100:
+ case 417:
strm.write_format("HTTP/1.1 %d %s\r\n\r\n", status,
- status_message(status));
+ detail::status_message(status));
break;
default: return write_response(strm, close_connection, req, res);
}
}
- // Routing
- auto routed = false;
+ // Rounting
+ bool routed = false;
#ifdef CPPHTTPLIB_NO_EXCEPTIONS
routed = routing(req, res, strm);
#else
@@ -6721,7 +6144,7 @@ Server::process_request(Stream &strm, bool close_connection,
exception_handler_(req, res, ep);
routed = true;
} else {
- res.status = StatusCode::InternalServerError_500;
+ res.status = 500;
std::string val;
auto s = e.what();
for (size_t i = 0; s[i]; i++) {
@@ -6739,29 +6162,17 @@ Server::process_request(Stream &strm, bool close_connection,
exception_handler_(req, res, ep);
routed = true;
} else {
- res.status = StatusCode::InternalServerError_500;
+ res.status = 500;
res.set_header("EXCEPTION_WHAT", "UNKNOWN");
}
}
#endif
- if (routed) {
- if (res.status == -1) {
- res.status = req.ranges.empty() ? StatusCode::OK_200
- : StatusCode::PartialContent_206;
- }
-
- if (detail::range_error(req, res)) {
- res.body.clear();
- res.content_length_ = 0;
- res.content_provider_ = nullptr;
- res.status = StatusCode::RangeNotSatisfiable_416;
- return write_response(strm, close_connection, req, res);
- }
+ if (routed) {
+ if (res.status == -1) { res.status = req.ranges.empty() ? 200 : 206; }
return write_response_with_content(strm, close_connection, req, res);
} else {
- if (res.status == -1) { res.status = StatusCode::NotFound_404; }
-
+ if (res.status == -1) { res.status = 404; }
return write_response(strm, close_connection, req, res);
}
}
@@ -6861,7 +6272,7 @@ inline socket_t ClientImpl::create_client_socket(Error &error) const {
// Check is custom IP specified for host_
std::string ip;
auto it = addr_map_.find(host_);
- if (it != addr_map_.end()) { ip = it->second; }
+ if (it != addr_map_.end()) ip = it->second;
return detail::create_client_socket(
host_, ip, port_, address_family_, tcp_nodelay_, socket_options_,
@@ -6886,7 +6297,7 @@ inline void ClientImpl::shutdown_ssl(Socket & /*socket*/,
socket_requests_are_from_thread_ == std::this_thread::get_id());
}
-inline void ClientImpl::shutdown_socket(Socket &socket) const {
+inline void ClientImpl::shutdown_socket(Socket &socket) {
if (socket.sock == INVALID_SOCKET) { return; }
detail::shutdown_socket(socket.sock);
}
@@ -6911,7 +6322,7 @@ inline void ClientImpl::close_socket(Socket &socket) {
}
inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,
- Response &res) const {
+ Response &res) {
std::array<char, 2048> buf{};
detail::stream_line_reader line_reader(strm, buf.data(), buf.size());
@@ -6919,9 +6330,9 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,
if (!line_reader.getline()) { return false; }
#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR
- const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n");
-#else
const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n");
+#else
+ const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n");
#endif
std::cmatch m;
@@ -6933,7 +6344,7 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,
res.reason = std::string(m[3]);
// Ignore '100 Continue'
- while (res.status == StatusCode::Continue_100) {
+ while (res.status == 100) {
if (!line_reader.getline()) { return false; } // CRLF
if (!line_reader.getline()) { return false; } // next response line
@@ -7081,31 +6492,15 @@ inline bool ClientImpl::handle_request(Stream &strm, Request &req,
if (!ret) { return false; }
- if (res.get_header_value("Connection") == "close" ||
- (res.version == "HTTP/1.0" && res.reason != "Connection established")) {
- // TODO this requires a not-entirely-obvious chain of calls to be correct
- // for this to be safe.
-
- // This is safe to call because handle_request is only called by send_
- // which locks the request mutex during the process. It would be a bug
- // to call it from a different thread since it's a thread-safety issue
- // to do these things to the socket if another thread is using the socket.
- std::lock_guard<std::mutex> guard(socket_mutex_);
- shutdown_ssl(socket_, true);
- shutdown_socket(socket_);
- close_socket(socket_);
- }
-
if (300 < res.status && res.status < 400 && follow_location_) {
req = req_save;
ret = redirect(req, res, error);
}
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- if ((res.status == StatusCode::Unauthorized_401 ||
- res.status == StatusCode::ProxyAuthenticationRequired_407) &&
+ if ((res.status == 401 || res.status == 407) &&
req.authorization_count_ < 5) {
- auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407;
+ auto is_proxy = res.status == 407;
const auto &username =
is_proxy ? proxy_digest_auth_username_ : digest_auth_username_;
const auto &password =
@@ -7176,7 +6571,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
} else {
if (next_scheme == "https") {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- SSLClient cli(next_host, next_port);
+ SSLClient cli(next_host.c_str(), next_port);
cli.copy_settings(*this);
if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); }
return detail::redirect(cli, req, res, path, location, error);
@@ -7184,7 +6579,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
return false;
#endif
} else {
- ClientImpl cli(next_host, next_port);
+ ClientImpl cli(next_host.c_str(), next_port);
cli.copy_settings(*this);
return detail::redirect(cli, req, res, path, location, error);
}
@@ -7193,7 +6588,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
inline bool ClientImpl::write_content_with_provider(Stream &strm,
const Request &req,
- Error &error) const {
+ Error &error) {
auto is_shutting_down = []() { return false; };
if (req.is_chunked_content_provider_) {
@@ -7221,32 +6616,32 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,
// Prepare additional headers
if (close_connection) {
if (!req.has_header("Connection")) {
- req.set_header("Connection", "close");
+ req.headers.emplace("Connection", "close");
}
}
if (!req.has_header("Host")) {
if (is_ssl()) {
if (port_ == 443) {
- req.set_header("Host", host_);
+ req.headers.emplace("Host", host_);
} else {
- req.set_header("Host", host_and_port_);
+ req.headers.emplace("Host", host_and_port_);
}
} else {
if (port_ == 80) {
- req.set_header("Host", host_);
+ req.headers.emplace("Host", host_);
} else {
- req.set_header("Host", host_and_port_);
+ req.headers.emplace("Host", host_and_port_);
}
}
}
- if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); }
+ if (!req.has_header("Accept")) { req.headers.emplace("Accept", "*/*"); }
#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT
if (!req.has_header("User-Agent")) {
auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION;
- req.set_header("User-Agent", agent);
+ req.headers.emplace("User-Agent", agent);
}
#endif
@@ -7255,23 +6650,23 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,
if (!req.is_chunked_content_provider_) {
if (!req.has_header("Content-Length")) {
auto length = std::to_string(req.content_length_);
- req.set_header("Content-Length", length);
+ req.headers.emplace("Content-Length", length);
}
}
} else {
if (req.method == "POST" || req.method == "PUT" ||
req.method == "PATCH") {
- req.set_header("Content-Length", "0");
+ req.headers.emplace("Content-Length", "0");
}
}
} else {
if (!req.has_header("Content-Type")) {
- req.set_header("Content-Type", "text/plain");
+ req.headers.emplace("Content-Type", "text/plain");
}
if (!req.has_header("Content-Length")) {
auto length = std::to_string(req.body.size());
- req.set_header("Content-Length", length);
+ req.headers.emplace("Content-Length", length);
}
}
@@ -7311,7 +6706,7 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,
const auto &path = url_encode_ ? detail::encode_url(req.path) : req.path;
bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str());
- header_writer_(bstrm, req.headers);
+ detail::write_headers(bstrm, req.headers);
// Flush buffer
auto &data = bstrm.get_buffer();
@@ -7339,10 +6734,12 @@ inline std::unique_ptr<Response> ClientImpl::send_with_content_provider(
ContentProvider content_provider,
ContentProviderWithoutLength content_provider_without_length,
const std::string &content_type, Error &error) {
- if (!content_type.empty()) { req.set_header("Content-Type", content_type); }
+ if (!content_type.empty()) {
+ req.headers.emplace("Content-Type", content_type);
+ }
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
- if (compress_) { req.set_header("Content-Encoding", "gzip"); }
+ if (compress_) { req.headers.emplace("Content-Encoding", "gzip"); }
#endif
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
@@ -7403,9 +6800,10 @@ inline std::unique_ptr<Response> ClientImpl::send_with_content_provider(
req.content_provider_ = detail::ContentProviderAdapter(
std::move(content_provider_without_length));
req.is_chunked_content_provider_ = true;
- req.set_header("Transfer-Encoding", "chunked");
+ req.headers.emplace("Transfer-Encoding", "chunked");
} else {
req.body.assign(body, content_length);
+ ;
}
}
@@ -7466,8 +6864,7 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,
}
// Body
- if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" &&
- req.method != "CONNECT") {
+ if ((res.status != 204) && req.method != "HEAD" && req.method != "CONNECT") {
auto redirect = 300 < res.status && res.status < 400 && follow_location_;
if (req.response_handler && !redirect) {
@@ -7512,6 +6909,24 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,
}
}
+ if (res.get_header_value("Connection") == "close" ||
+ (res.version == "HTTP/1.0" && res.reason != "Connection established")) {
+ // TODO this requires a not-entirely-obvious chain of calls to be correct
+ // for this to be safe. Maybe a code refactor (such as moving this out to
+ // the send function and getting rid of the recursiveness of the mutex)
+ // could make this more obvious.
+
+ // This is safe to call because process_request is only called by
+ // handle_request which is only called by send, which locks the request
+ // mutex during the process. It would be a bug to call it from a different
+ // thread since it's a thread-safety issue to do these things to the socket
+ // if another thread is using the socket.
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+ shutdown_ssl(socket_, true);
+ shutdown_socket(socket_);
+ close_socket(socket_);
+ }
+
// Log
if (logger_) { logger_(req, res); }
@@ -7520,14 +6935,13 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,
inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(
const std::string &boundary, const MultipartFormDataItems &items,
- const MultipartFormDataProviderItems &provider_items) const {
- size_t cur_item = 0;
- size_t cur_start = 0;
+ const MultipartFormDataProviderItems &provider_items) {
+ size_t cur_item = 0, cur_start = 0;
// cur_item and cur_start are copied to within the std::function and maintain
// state between successive calls
return [&, cur_item, cur_start](size_t offset,
DataSink &sink) mutable -> bool {
- if (!offset && !items.empty()) {
+ if (!offset && items.size()) {
sink.os << detail::serialize_multipart_formdata(items, boundary, false);
return true;
} else if (cur_item < provider_items.size()) {
@@ -7540,13 +6954,12 @@ inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(
}
DataSink cur_sink;
- auto has_data = true;
+ bool has_data = true;
cur_sink.write = sink.write;
cur_sink.done = [&]() { has_data = false; };
- if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) {
+ if (!provider_items[cur_item].provider(offset - cur_start, cur_sink))
return false;
- }
if (!has_data) {
sink.os << detail::serialize_multipart_formdata_item_end();
@@ -7665,15 +7078,14 @@ inline Result ClientImpl::Get(const std::string &path, const Params &params,
if (params.empty()) { return Get(path, headers); }
std::string path_with_query = append_query_params(path, params);
- return Get(path_with_query, headers, std::move(progress));
+ return Get(path_with_query.c_str(), headers, progress);
}
inline Result ClientImpl::Get(const std::string &path, const Params &params,
const Headers &headers,
ContentReceiver content_receiver,
Progress progress) {
- return Get(path, params, headers, nullptr, std::move(content_receiver),
- std::move(progress));
+ return Get(path, params, headers, nullptr, content_receiver, progress);
}
inline Result ClientImpl::Get(const std::string &path, const Params &params,
@@ -7682,13 +7094,12 @@ inline Result ClientImpl::Get(const std::string &path, const Params &params,
ContentReceiver content_receiver,
Progress progress) {
if (params.empty()) {
- return Get(path, headers, std::move(response_handler),
- std::move(content_receiver), std::move(progress));
+ return Get(path, headers, response_handler, content_receiver, progress);
}
std::string path_with_query = append_query_params(path, params);
- return Get(path_with_query, headers, std::move(response_handler),
- std::move(content_receiver), std::move(progress));
+ return Get(path_with_query.c_str(), headers, response_handler,
+ content_receiver, progress);
}
inline Result ClientImpl::Head(const std::string &path) {
@@ -7790,7 +7201,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
const auto &content_type =
detail::serialize_multipart_formdata_get_content_type(boundary);
const auto &body = detail::serialize_multipart_formdata(items, boundary);
- return Post(path, headers, body, content_type);
+ return Post(path, headers, body, content_type.c_str());
}
inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
@@ -7803,7 +7214,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
const auto &content_type =
detail::serialize_multipart_formdata_get_content_type(boundary);
const auto &body = detail::serialize_multipart_formdata(items, boundary);
- return Post(path, headers, body, content_type);
+ return Post(path, headers, body, content_type.c_str());
}
inline Result
@@ -8012,7 +7423,9 @@ inline Result ClientImpl::Delete(const std::string &path,
req.headers = headers;
req.path = path;
- if (!content_type.empty()) { req.set_header("Content-Type", content_type); }
+ if (!content_type.empty()) {
+ req.headers.emplace("Content-Type", content_type);
+ }
req.body.assign(body, content_length);
return send_(std::move(req));
@@ -8045,6 +7458,13 @@ inline Result ClientImpl::Options(const std::string &path,
return send_(std::move(req));
}
+inline size_t ClientImpl::is_socket_open() const {
+ std::lock_guard<std::mutex> guard(socket_mutex_);
+ return socket_.is_open();
+}
+
+inline socket_t ClientImpl::socket() const { return socket_.sock; }
+
inline void ClientImpl::stop() {
std::lock_guard<std::mutex> guard(socket_mutex_);
@@ -8068,17 +7488,6 @@ inline void ClientImpl::stop() {
close_socket(socket_);
}
-inline std::string ClientImpl::host() const { return host_; }
-
-inline int ClientImpl::port() const { return port_; }
-
-inline size_t ClientImpl::is_socket_open() const {
- std::lock_guard<std::mutex> guard(socket_mutex_);
- return socket_.is_open();
-}
-
-inline socket_t ClientImpl::socket() const { return socket_.sock; }
-
inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) {
connection_timeout_sec_ = sec;
connection_timeout_usec_ = usec;
@@ -8127,11 +7536,6 @@ inline void ClientImpl::set_default_headers(Headers headers) {
default_headers_ = std::move(headers);
}
-inline void ClientImpl::set_header_writer(
- std::function<ssize_t(Stream &, Headers &)> const &writer) {
- header_writer_ = writer;
-}
-
inline void ClientImpl::set_address_family(int family) {
address_family_ = family;
}
@@ -8171,7 +7575,9 @@ inline void ClientImpl::set_proxy_digest_auth(const std::string &username,
proxy_digest_auth_username_ = username;
proxy_digest_auth_password_ = password;
}
+#endif
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
inline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path,
const std::string &ca_cert_dir_path) {
ca_cert_file_path_ = ca_cert_file_path;
@@ -8183,34 +7589,9 @@ inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) {
ca_cert_store_ = ca_cert_store;
}
}
+#endif
-inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert,
- std::size_t size) const {
- auto mem = BIO_new_mem_buf(ca_cert, static_cast<int>(size));
- if (!mem) { return nullptr; }
-
- auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr);
- if (!inf) {
- BIO_free_all(mem);
- return nullptr;
- }
-
- auto cts = X509_STORE_new();
- if (cts) {
- for (auto i = 0; i < static_cast<int>(sk_X509_INFO_num(inf)); i++) {
- auto itmp = sk_X509_INFO_value(inf, i);
- if (!itmp) { continue; }
-
- if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); }
- if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); }
- }
- }
-
- sk_X509_INFO_pop_free(inf, X509_INFO_free);
- BIO_free_all(mem);
- return cts;
-}
-
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
inline void ClientImpl::enable_server_certificate_verification(bool enabled) {
server_certificate_verification_ = enabled;
}
@@ -8274,7 +7655,7 @@ bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl,
U ssl_connect_or_accept,
time_t timeout_sec,
time_t timeout_usec) {
- auto res = 0;
+ int res = 0;
while ((res = ssl_connect_or_accept(ssl)) != 1) {
auto err = SSL_get_error(ssl, res);
switch (err) {
@@ -8337,7 +7718,7 @@ inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl,
SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY);
}
-inline SSLSocketStream::~SSLSocketStream() = default;
+inline SSLSocketStream::~SSLSocketStream() {}
inline bool SSLSocketStream::is_readable() const {
return detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;
@@ -8355,7 +7736,7 @@ inline ssize_t SSLSocketStream::read(char *ptr, size_t size) {
auto ret = SSL_read(ssl_, ptr, static_cast<int>(size));
if (ret < 0) {
auto err = SSL_get_error(ssl_, ret);
- auto n = 1000;
+ int n = 1000;
#ifdef _WIN32
while (--n >= 0 && (err == SSL_ERROR_WANT_READ ||
(err == SSL_ERROR_SYSCALL &&
@@ -8388,7 +7769,7 @@ inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) {
auto ret = SSL_write(ssl_, ptr, static_cast<int>(handle_size));
if (ret < 0) {
auto err = SSL_get_error(ssl_, ret);
- auto n = 1000;
+ int n = 1000;
#ifdef _WIN32
while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE ||
(err == SSL_ERROR_SYSCALL &&
@@ -8441,10 +7822,10 @@ inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path,
SSL_CTX_set_min_proto_version(ctx_, TLS1_1_VERSION);
+ // add default password callback before opening encrypted private key
if (private_key_password != nullptr && (private_key_password[0] != '\0')) {
- SSL_CTX_set_default_passwd_cb_userdata(
- ctx_,
- reinterpret_cast<void *>(const_cast<char *>(private_key_password)));
+ SSL_CTX_set_default_passwd_cb_userdata(ctx_,
+ (char *)private_key_password);
}
if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 ||
@@ -8546,23 +7927,16 @@ inline SSLClient::SSLClient(const std::string &host, int port)
inline SSLClient::SSLClient(const std::string &host, int port,
const std::string &client_cert_path,
- const std::string &client_key_path,
- const std::string &private_key_password)
+ const std::string &client_key_path)
: ClientImpl(host, port, client_cert_path, client_key_path) {
ctx_ = SSL_CTX_new(TLS_client_method());
detail::split(&host_[0], &host_[host_.size()], '.',
[&](const char *b, const char *e) {
- host_components_.emplace_back(b, e);
+ host_components_.emplace_back(std::string(b, e));
});
if (!client_cert_path.empty() && !client_key_path.empty()) {
- if (!private_key_password.empty()) {
- SSL_CTX_set_default_passwd_cb_userdata(
- ctx_, reinterpret_cast<void *>(
- const_cast<char *>(private_key_password.c_str())));
- }
-
if (SSL_CTX_use_certificate_file(ctx_, client_cert_path.c_str(),
SSL_FILETYPE_PEM) != 1 ||
SSL_CTX_use_PrivateKey_file(ctx_, client_key_path.c_str(),
@@ -8574,23 +7948,16 @@ inline SSLClient::SSLClient(const std::string &host, int port,
}
inline SSLClient::SSLClient(const std::string &host, int port,
- X509 *client_cert, EVP_PKEY *client_key,
- const std::string &private_key_password)
+ X509 *client_cert, EVP_PKEY *client_key)
: ClientImpl(host, port) {
ctx_ = SSL_CTX_new(TLS_client_method());
detail::split(&host_[0], &host_[host_.size()], '.',
[&](const char *b, const char *e) {
- host_components_.emplace_back(b, e);
+ host_components_.emplace_back(std::string(b, e));
});
if (client_cert != nullptr && client_key != nullptr) {
- if (!private_key_password.empty()) {
- SSL_CTX_set_default_passwd_cb_userdata(
- ctx_, reinterpret_cast<void *>(
- const_cast<char *>(private_key_password.c_str())));
- }
-
if (SSL_CTX_use_certificate(ctx_, client_cert) != 1 ||
SSL_CTX_use_PrivateKey(ctx_, client_key) != 1) {
SSL_CTX_free(ctx_);
@@ -8622,11 +7989,6 @@ inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) {
}
}
-inline void SSLClient::load_ca_cert_store(const char *ca_cert,
- std::size_t size) {
- set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size));
-}
-
inline long SSLClient::get_openssl_verify_result() const {
return verify_result_;
}
@@ -8641,14 +8003,14 @@ inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) {
inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,
bool &success, Error &error) {
success = true;
- Response proxy_res;
+ Response res2;
if (!detail::process_client_socket(
socket.sock, read_timeout_sec_, read_timeout_usec_,
write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {
Request req2;
req2.method = "CONNECT";
req2.path = host_and_port_;
- return process_request(strm, req2, proxy_res, false, error);
+ return process_request(strm, req2, res2, false, error);
})) {
// Thread-safe to close everything because we are assuming there are no
// requests in flight
@@ -8659,12 +8021,12 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,
return false;
}
- if (proxy_res.status == StatusCode::ProxyAuthenticationRequired_407) {
+ if (res2.status == 407) {
if (!proxy_digest_auth_username_.empty() &&
!proxy_digest_auth_password_.empty()) {
std::map<std::string, std::string> auth;
- if (detail::parse_www_authenticate(proxy_res, auth, true)) {
- proxy_res = Response();
+ if (detail::parse_www_authenticate(res2, auth, true)) {
+ Response res3;
if (!detail::process_client_socket(
socket.sock, read_timeout_sec_, read_timeout_usec_,
write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {
@@ -8675,7 +8037,7 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,
req3, auth, 1, detail::random_string(10),
proxy_digest_auth_username_, proxy_digest_auth_password_,
true));
- return process_request(strm, req3, proxy_res, false, error);
+ return process_request(strm, req3, res3, false, error);
})) {
// Thread-safe to close everything because we are assuming there are
// no requests in flight
@@ -8686,28 +8048,17 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,
return false;
}
}
+ } else {
+ res = res2;
+ return false;
}
}
- // If status code is not 200, proxy request is failed.
- // Set error to ProxyConnection and return proxy response
- // as the response of the request
- if (proxy_res.status != StatusCode::OK_200) {
- error = Error::ProxyConnection;
- res = std::move(proxy_res);
- // Thread-safe to close everything because we are assuming there are
- // no requests in flight
- shutdown_ssl(socket, true);
- shutdown_socket(socket);
- close_socket(socket);
- return false;
- }
-
return true;
}
inline bool SSLClient::load_certs() {
- auto ret = true;
+ bool ret = true;
std::call_once(initialize_cert_, [&]() {
std::lock_guard<std::mutex> guard(ctx_mutex_);
@@ -8783,11 +8134,7 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) {
return true;
},
[&](SSL *ssl2) {
- // NOTE: Direct call instead of using the OpenSSL macro to suppress
- // -Wold-style-cast warning
- // SSL_set_tlsext_host_name(ssl2, host_.c_str());
- SSL_ctrl(ssl2, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name,
- static_cast<void *>(const_cast<char *>(host_.c_str())));
+ SSL_set_tlsext_host_name(ssl2, host_.c_str());
return true;
});
@@ -8861,8 +8208,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {
auto type = GEN_DNS;
- struct in6_addr addr6 {};
- struct in_addr addr {};
+ struct in6_addr addr6;
+ struct in_addr addr;
size_t addr_len = 0;
#ifndef __MINGW32__
@@ -8887,9 +8234,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {
for (decltype(count) i = 0; i < count && !dsn_matched; i++) {
auto val = sk_GENERAL_NAME_value(alt_names, i);
if (val->type == type) {
- auto name =
- reinterpret_cast<const char *>(ASN1_STRING_get0_data(val->d.ia5));
- auto name_len = static_cast<size_t>(ASN1_STRING_length(val->d.ia5));
+ auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5);
+ auto name_len = (size_t)ASN1_STRING_length(val->d.ia5);
switch (type) {
case GEN_DNS: dsn_matched = check_host_name(name, name_len); break;
@@ -8907,8 +8253,7 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {
if (dsn_matched || ip_matched) { ret = true; }
}
- GENERAL_NAMES_free(const_cast<STACK_OF(GENERAL_NAME) *>(
- reinterpret_cast<const STACK_OF(GENERAL_NAME) *>(alt_names)));
+ GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names);
return ret;
}
@@ -8937,7 +8282,7 @@ inline bool SSLClient::check_host_name(const char *pattern,
std::vector<std::string> pattern_components;
detail::split(&pattern[0], &pattern[pattern_len], '.',
[&](const char *b, const char *e) {
- pattern_components.emplace_back(b, e);
+ pattern_components.emplace_back(std::string(b, e));
});
if (host_components_.size() != pattern_components.size()) { return false; }
@@ -9016,7 +8361,7 @@ inline Client::Client(const std::string &host, int port,
: cli_(detail::make_unique<ClientImpl>(host, port, client_cert_path,
client_key_path)) {}
-inline Client::~Client() = default;
+inline Client::~Client() {}
inline bool Client::is_valid() const {
return cli_ != nullptr && cli_->is_valid();
@@ -9076,20 +8421,19 @@ inline Result Client::Get(const std::string &path, const Headers &headers,
}
inline Result Client::Get(const std::string &path, const Params &params,
const Headers &headers, Progress progress) {
- return cli_->Get(path, params, headers, std::move(progress));
+ return cli_->Get(path, params, headers, progress);
}
inline Result Client::Get(const std::string &path, const Params &params,
const Headers &headers,
ContentReceiver content_receiver, Progress progress) {
- return cli_->Get(path, params, headers, std::move(content_receiver),
- std::move(progress));
+ return cli_->Get(path, params, headers, content_receiver, progress);
}
inline Result Client::Get(const std::string &path, const Params &params,
const Headers &headers,
ResponseHandler response_handler,
ContentReceiver content_receiver, Progress progress) {
- return cli_->Get(path, params, headers, std::move(response_handler),
- std::move(content_receiver), std::move(progress));
+ return cli_->Get(path, params, headers, response_handler, content_receiver,
+ progress);
}
inline Result Client::Head(const std::string &path) { return cli_->Head(path); }
@@ -9321,16 +8665,12 @@ inline bool Client::send(Request &req, Response &res, Error &error) {
inline Result Client::send(const Request &req) { return cli_->send(req); }
-inline void Client::stop() { cli_->stop(); }
-
-inline std::string Client::host() const { return cli_->host(); }
-
-inline int Client::port() const { return cli_->port(); }
-
inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); }
inline socket_t Client::socket() const { return cli_->socket(); }
+inline void Client::stop() { cli_->stop(); }
+
inline void
Client::set_hostname_addr_map(std::map<std::string, std::string> addr_map) {
cli_->set_hostname_addr_map(std::move(addr_map));
@@ -9340,11 +8680,6 @@ inline void Client::set_default_headers(Headers headers) {
cli_->set_default_headers(std::move(headers));
}
-inline void Client::set_header_writer(
- std::function<ssize_t(Stream &, Headers &)> const &writer) {
- cli_->set_header_writer(writer);
-}
-
inline void Client::set_address_family(int family) {
cli_->set_address_family(family);
}
@@ -9419,9 +8754,7 @@ inline void Client::enable_server_certificate_verification(bool enabled) {
}
#endif
-inline void Client::set_logger(Logger logger) {
- cli_->set_logger(std::move(logger));
-}
+inline void Client::set_logger(Logger logger) { cli_->set_logger(logger); }
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
inline void Client::set_ca_cert_path(const std::string &ca_cert_file_path,
@@ -9437,10 +8770,6 @@ inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) {
}
}
-inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) {
- set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size));
-}
-
inline long Client::get_openssl_verify_result() const {
if (is_ssl_) {
return static_cast<SSLClient &>(*cli_).get_openssl_verify_result();
diff --git a/llama.cpp/server/oai.h b/llama.cpp/server/oai.h
new file mode 100644
index 00000000..cdbff727
--- /dev/null
+++ b/llama.cpp/server/oai.h
@@ -0,0 +1,228 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <set>
+#include <mutex>
+#include <condition_variable>
+#include <unordered_map>
+
+#include "llama.cpp/json.h"
+#include "utils.h"
+
+#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"
+
+using json = nlohmann::json;
+
+inline static json oaicompat_completion_params_parse(
+ const struct llama_model * model,
+ const json &body, /* openai api json semantics */
+ const std::string &chat_template)
+{
+ json llama_params;
+
+ llama_params["__oaicompat"] = true;
+
+ // Map OpenAI parameters to llama.cpp parameters
+ //
+ // For parameters that are defined by the OpenAI documentation (e.g.
+ // temperature), we explicitly specify OpenAI's intended default; we
+ // need to do that because sometimes OpenAI disagrees with llama.cpp
+ //
+ // https://platform.openai.com/docs/api-reference/chat/create
+ llama_sampling_params default_sparams;
+ llama_params["model"] = json_value(body, "model", std::string("unknown"));
+ llama_params["prompt"] = format_chat(model, chat_template, body["messages"]);
+ llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);
+ llama_params["temperature"] = json_value(body, "temperature", 0.0);
+ llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);
+ llama_params["top_p"] = json_value(body, "top_p", 1.0);
+ llama_params["n_predict"] = json_value(body, "max_tokens", -1);
+ llama_params["logit_bias"] = json_value(body, "logit_bias",json::object());
+ llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);
+ llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);
+ llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);
+ llama_params["stream"] = json_value(body, "stream", false);
+ llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);
+ llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);
+ llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);
+ llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);
+ llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);
+ llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);
+ llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);
+ llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);
+
+ if (body.count("grammar") != 0) {
+ llama_params["grammar"] = json_value(body, "grammar", json::object());
+ }
+
+ // Handle 'stop' field
+ if (body.contains("stop") && body["stop"].is_string()) {
+ llama_params["stop"] = json::array({body["stop"].get<std::string>()});
+ } else {
+ llama_params["stop"] = json_value(body, "stop", json::array());
+ }
+
+ // Ensure there is ChatML-specific end sequence among stop words
+ llama_params["stop"].push_back("<|im_end|>");
+
+ return llama_params;
+}
+
+inline static json format_final_response_oaicompat(const json &request, const task_result &response, bool streaming = false)
+{
+ json result = response.result_json;
+
+ bool stopped_word = result.count("stopped_word") != 0;
+ bool stopped_eos = json_value(result, "stopped_eos", false);
+ int num_tokens_predicted = json_value(result, "tokens_predicted", 0);
+ int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);
+ std::string content = json_value(result, "content", std::string(""));
+
+ std::string finish_reason = "length";
+ if (stopped_word || stopped_eos) {
+ finish_reason = "stop";
+ }
+
+ json choices =
+ streaming ? json::array({json{{"finish_reason", finish_reason},
+ {"index", 0},
+ {"delta", json::object()}}})
+ : json::array({json{{"finish_reason", finish_reason},
+ {"index", 0},
+ {"message", json{{"content", content},
+ {"role", "assistant"}}}}});
+
+ std::time_t t = std::time(0);
+
+ json res =
+ json{{"choices", choices},
+ {"created", t},
+ {"model",
+ json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
+ {"object", streaming ? "chat.completion.chunk" : "chat.completion"},
+ {"usage",
+ json{{"completion_tokens", num_tokens_predicted},
+ {"prompt_tokens", num_prompt_tokens},
+ {"total_tokens", num_tokens_predicted + num_prompt_tokens}}},
+ {"id", gen_chatcmplid()}};
+
+ if (server_verbose) {
+ res["__verbose"] = result;
+ }
+
+ if (result.contains("completion_probabilities")) {
+ res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());
+ }
+
+ return res;
+}
+
+// return value is vector as there is one case where we might need to generate two responses
+inline static std::vector<json> format_partial_response_oaicompat(const task_result &response) {
+ json result = response.result_json;
+
+ if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {
+ return std::vector<json>({response.result_json});
+ }
+
+ bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;
+ std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));
+
+ bool stopped_word = json_value(result, "stopped_word", false);
+ bool stopped_eos = json_value(result, "stopped_eos", false);
+ bool stopped_limit = json_value(result, "stopped_limit", false);
+ std::string content = json_value(result, "content", std::string(""));
+
+ std::string finish_reason;
+ if (stopped_word || stopped_eos) {
+ finish_reason = "stop";
+ }
+ if (stopped_limit) {
+ finish_reason = "length";
+ }
+
+ std::time_t t = std::time(0);
+
+ json choices;
+
+ if (!finish_reason.empty()) {
+ choices = json::array({json{{"finish_reason", finish_reason},
+ {"index", 0},
+ {"delta", json::object()}}});
+ } else {
+ if (first) {
+ if (content.empty()) {
+ choices = json::array({json{{"finish_reason", nullptr},
+ {"index", 0},
+ {"delta", json{{"role", "assistant"}}}}});
+ } else {
+ // We have to send this as two updates to conform to openai behavior
+ json initial_ret = json{{"choices", json::array({json{
+ {"finish_reason", nullptr},
+ {"index", 0},
+ {"delta", json{
+ {"role", "assistant"}
+ }}}})},
+ {"created", t},
+ {"id", gen_chatcmplid()},
+ {"model", modelname},
+ {"object", "chat.completion.chunk"}};
+
+ json second_ret = json{
+ {"choices", json::array({json{{"finish_reason", nullptr},
+ {"index", 0},
+ {"delta", json{
+ {"content", content}}}
+ }})},
+ {"created", t},
+ {"id", gen_chatcmplid()},
+ {"model", modelname},
+ {"object", "chat.completion.chunk"}};
+
+ return std::vector<json>({initial_ret, second_ret});
+ }
+ } else {
+ // Some idiosyncrasy in task processing logic makes several trailing calls
+ // with empty content, we ignore these at the calee site.
+ if (content.empty()) {
+ return std::vector<json>({json::object()});
+ }
+
+ choices = json::array({json{
+ {"finish_reason", nullptr},
+ {"index", 0},
+ {"delta",
+ json{
+ {"content", content},
+ }},
+ }});
+ }
+ }
+
+ json ret = json{{"choices", choices},
+ {"created", t},
+ {"id", gen_chatcmplid()},
+ {"model", modelname},
+ {"object", "chat.completion.chunk"}};
+
+ return std::vector<json>({ret});
+}
+
+inline static json format_embeddings_response_oaicompat(const json &request, const json &embeddings)
+{
+ json res =
+ json{
+ {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
+ {"object", "list"},
+ {"usage",
+ json{{"prompt_tokens", 0},
+ {"total_tokens", 0}}},
+ {"data", embeddings}
+ };
+ return res;
+}
+
diff --git a/llama.cpp/server/public/completion.js b/llama.cpp/server/public/completion.js
index 835ce6e6..ab38a7b4 100644
--- a/llama.cpp/server/public/completion.js
+++ b/llama.cpp/server/public/completion.js
@@ -96,18 +96,18 @@ export async function* llama(prompt, params = {}, config = {}) {
}
}
if (result.error) {
- try {
- result.error = JSON.parse(result.error);
- if (result.error.message.includes('slot unavailable')) {
- // Throw an error to be caught by upstream callers
- throw new Error('slot unavailable');
- } else {
- console.error(`llama.cpp error [${result.error.code} - ${result.error.type}]: ${result.error.message}`);
- }
- } catch(e) {
- console.error(`llama.cpp error ${result.error}`)
+ result.error = JSON.parse(result.error);
+ if (result.error.content.includes('slot unavailable')) {
+ // Throw an error to be caught by upstream callers
+ throw new Error('slot unavailable');
+ } else {
+ console.error(`llama.cpp error: ${result.error.content}`);
}
}
+ if (result.error) {
+ result.error = JSON.parse(result.error);
+ console.error(`llama.cpp error: ${result.error.content}`);
+ }
}
}
}
diff --git a/llama.cpp/server/public/index.html b/llama.cpp/server/public/index.html
index 18f95f19..47cb5f76 100644
--- a/llama.cpp/server/public/index.html
+++ b/llama.cpp/server/public/index.html
@@ -630,16 +630,14 @@
const grammarJsonSchemaPropOrder = signal('')
const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value
- const convertJSONSchemaGrammar = async () => {
+ const convertJSONSchemaGrammar = () => {
try {
- let schema = JSON.parse(params.value.grammar)
- const converter = new SchemaConverter({
- prop_order: grammarJsonSchemaPropOrder.value
+ const schema = JSON.parse(params.value.grammar)
+ const converter = new SchemaConverter(
+ grammarJsonSchemaPropOrder.value
.split(',')
- .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),
- allow_fetch: true,
- })
- schema = await converter.resolveRefs(schema, 'input')
+ .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {})
+ )
converter.visit(schema, '')
params.value = {
...params.value,
diff --git a/llama.cpp/server/public/index.js b/llama.cpp/server/public/index.js
index aa3ffb5e..9db5a9d9 100644
--- a/llama.cpp/server/public/index.js
+++ b/llama.cpp/server/public/index.js
@@ -1 +1 @@
-function t(){throw new Error("Cycle detected")}const n=Symbol.for("preact-signals");function e(){if(f>1){f--;return}let t,n=!1;while(void 0!==o){let _=o;o=void 0;l++;while(void 0!==_){const i=_.o;_.o=void 0;_.f&=-3;if(!(8&_.f)&&p(_))try{_.c()}catch(e){if(!n){t=e;n=!0}}_=i}}l=0;f--;if(n)throw t}function _(t){if(f>0)return t();f++;try{return t()}finally{e()}}let i,o,r=0;function u(t){if(r>0)return t();const n=i;i=void 0;r++;try{return t()}finally{r--;i=n}}let f=0,l=0,s=0;function c(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function h(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}h.prototype.brand=n;h.prototype.h=function(){return!0};h.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};h.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};h.prototype.subscribe=function(t){const n=this;return w((function(){const e=n.value,_=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=_}}))};h.prototype.valueOf=function(){return this.value};h.prototype.toString=function(){return this.value+""};h.prototype.toJSON=function(){return this.value};h.prototype.peek=function(){return this.v};Object.defineProperty(h.prototype,"value",{get(){const t=c(this);if(void 0!==t)t.i=this.i;return this.v},set(n){if(i instanceof y)!function(){throw new Error("Computed cannot have side-effects")}();if(n!==this.v){if(l>100)t();this.v=n;this.i++;s++;f++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{e()}}}});function a(t){return new h(t)}function p(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function d(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function v(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function y(t){h.call(this,void 0);this.x=t;this.s=void 0;this.g=s-1;this.f=4}(y.prototype=new h).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===s)return!0;this.g=s;this.f|=1;if(this.i>0&&!p(this)){this.f&=-2;return!0}const t=i;try{d(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;v(this);this.f&=-2;return!0};y.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}h.prototype.S.call(this,t)};y.prototype.U=function(t){if(void 0!==this.t){h.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};y.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};y.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(y.prototype,"value",{get(){if(1&this.f)t();const n=c(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function m(t){return new y(t)}function g(t){const n=t.u;t.u=void 0;if("function"==typeof n){f++;const _=i;i=void 0;try{n()}catch(n){t.f&=-2;t.f|=8;b(t);throw n}finally{i=_;e()}}}function b(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;g(t)}function k(t){if(i!==this)throw new Error("Out-of-order effect");v(this);i=t;this.f&=-2;if(8&this.f)b(this);e()}function S(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}S.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};S.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;g(this);d(this);f++;const n=i;i=this;return k.bind(this,n)};S.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=o;o=this}};S.prototype.d=function(){this.f|=8;if(!(1&this.f))b(this)};function w(t){const n=new S(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,C,E,U,H,P,N,$,D,T={},V=[],A=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,F=Array.isArray;function M(t,n){for(var e in n)t[e]=n[e];return t}function W(t){var n=t.parentNode;n&&n.removeChild(t)}function L(t,n,e){var _,i,o,r={};for(o in n)"key"==o?_=n[o]:"ref"==o?i=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,_,i,null)}function O(t,n,e,_,i){var o={type:t,props:n,key:e,ref:_,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,constructor:void 0,__v:null==i?++E:i,__i:-1,__u:0};return null==i&&null!=C.vnode&&C.vnode(o),o}function R(){return{current:null}}function j(t){return t.children}function I(t,n){this.props=t,this.context=n}function q(t,n){if(null==n)return t.__?q(t.__,t.__i+1):null;for(var e;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e)return e.__e;return"function"==typeof t.type?q(t):null}function B(t,n,e){var _,i=t.__v,o=i.__e,r=t.__P;if(r)return(_=M({},i)).__v=i.__v+1,C.vnode&&C.vnode(_),it(r,_,i,t.__n,void 0!==r.ownerSVGElement,32&i.__u?[o]:null,n,null==o?q(i):o,!!(32&i.__u),e),_.__v=i.__v,_.__.__k[_.__i]=_,_.__d=void 0,_.__e!=o&&G(_),_}function G(t){var n,e;if(null!=(t=t.__)&&null!=t.__c){for(t.__e=t.__c.base=null,n=0;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e){t.__e=t.__c.base=e.__e;break}return G(t)}}function z(t){(!t.__d&&(t.__d=!0)&&H.push(t)&&!J.__r++||P!==C.debounceRendering)&&((P=C.debounceRendering)||N)(J)}function J(){var t,n,e,_=[],i=[];for(H.sort($);t=H.shift();)t.__d&&(e=H.length,n=B(t,_,i)||n,0===e||H.length>e?(ot(_,n,i),i.length=_.length=0,n=void 0,H.sort($)):n&&C.__c&&C.__c(n,V));n&&ot(_,n,i),J.__r=0}function K(t,n,e,_,i,o,r,u,f,l,s){var c,h,a,p,d,v=_&&_.__k||V,y=n.length;for(e.__d=f,Q(e,n,v),f=e.__d,c=0;c<y;c++)null!=(a=e.__k[c])&&"boolean"!=typeof a&&"function"!=typeof a&&(h=-1===a.__i?T:v[a.__i]||T,a.__i=c,it(t,a,h,i,o,r,u,f,l,s),p=a.__e,a.ref&&h.ref!=a.ref&&(h.ref&&ut(h.ref,null,a),s.push(a.ref,a.__c||p,a)),null==d&&null!=p&&(d=p),65536&a.__u||h.__k===a.__k?f=X(a,f,t):"function"==typeof a.type&&void 0!==a.__d?f=a.__d:p&&(f=p.nextSibling),a.__d=void 0,a.__u&=-196609);e.__d=f,e.__e=d}function Q(t,n,e){var _,i,o,r,u,f=n.length,l=e.length,s=l,c=0;for(t.__k=[],_=0;_<f;_++)r=_+c,null!=(i=t.__k[_]=null==(i=n[_])||"boolean"==typeof i||"function"==typeof i?null:"string"==typeof i||"number"==typeof i||"bigint"==typeof i||i.constructor==String?O(null,i,null,null,null):F(i)?O(j,{children:i},null,null,null):void 0===i.constructor&&i.__b>0?O(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):i)?(i.__=t,i.__b=t.__b+1,u=Z(i,e,r,s),i.__i=u,o=null,-1!==u&&(s--,(o=e[u])&&(o.__u|=131072)),null==o||null===o.__v?(-1==u&&c--,"function"!=typeof i.type&&(i.__u|=65536)):u!==r&&(u===r+1?c++:u>r?s>f-r?c+=u-r:c--:u<r?u==r-1&&(c=u-r):c=0,u!==_+c&&(i.__u|=65536))):(o=e[r])&&null==o.key&&o.__e&&0==(131072&o.__u)&&(o.__e==t.__d&&(t.__d=q(o)),ft(o,o,!1),e[r]=null,s--);if(s)for(_=0;_<l;_++)null!=(o=e[_])&&0==(131072&o.__u)&&(o.__e==t.__d&&(t.__d=q(o)),ft(o,o))}function X(t,n,e){var _,i;if("function"==typeof t.type){for(_=t.__k,i=0;_&&i<_.length;i++)_[i]&&(_[i].__=t,n=X(_[i],n,e));return n}t.__e!=n&&(e.insertBefore(t.__e,n||null),n=t.__e);do{n=n&&n.nextSibling}while(null!=n&&8===n.nodeType);return n}function Y(t,n){return n=n||[],null==t||"boolean"==typeof t||(F(t)?t.some((function(t){Y(t,n)})):n.push(t)),n}function Z(t,n,e,_){var i=t.key,o=t.type,r=e-1,u=e+1,f=n[e];if(null===f||f&&i==f.key&&o===f.type&&0==(131072&f.__u))return e;if(_>(null!=f&&0==(131072&f.__u)?1:0))for(;r>=0||u<n.length;){if(r>=0){if((f=n[r])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return r;r--}if(u<n.length){if((f=n[u])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return u;u++}}return-1}function tt(t,n,e){"-"===n[0]?t.setProperty(n,null==e?"":e):t[n]=null==e?"":"number"!=typeof e||A.test(n)?e:e+"px"}function nt(t,n,e,_,i){var o;t:if("style"===n)if("string"==typeof e)t.style.cssText=e;else{if("string"==typeof _&&(t.style.cssText=_=""),_)for(n in _)e&&n in e||tt(t.style,n,"");if(e)for(n in e)_&&e[n]===_[n]||tt(t.style,n,e[n])}else if("o"===n[0]&&"n"===n[1])o=n!==(n=n.replace(/(PointerCapture)$|Capture$/i,"$1")),n=n.toLowerCase()in t?n.toLowerCase().slice(2):n.slice(2),t.l||(t.l={}),t.l[n+o]=e,e?_?e.u=_.u:(e.u=Date.now(),t.addEventListener(n,o?_t:et,o)):t.removeEventListener(n,o?_t:et,o);else{if(i)n=n.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if("width"!==n&&"height"!==n&&"href"!==n&&"list"!==n&&"form"!==n&&"tabIndex"!==n&&"download"!==n&&"rowSpan"!==n&&"colSpan"!==n&&"role"!==n&&n in t)try{t[n]=null==e?"":e;break t}catch(t){}"function"==typeof e||(null==e||!1===e&&"-"!==n[4]?t.removeAttribute(n):t.setAttribute(n,e))}}function et(t){if(this.l){var n=this.l[t.type+!1];if(t.t){if(t.t<=n.u)return}else t.t=Date.now();return n(C.event?C.event(t):t)}}function _t(t){if(this.l)return this.l[t.type+!0](C.event?C.event(t):t)}function it(t,n,e,_,i,o,r,u,f,l){var s,c,h,a,p,d,v,y,m,g,b,k,S,w,x,E=n.type;if(void 0!==n.constructor)return null;128&e.__u&&(f=!!(32&e.__u),o=[u=n.__e=e.__e]),(s=C.__b)&&s(n);t:if("function"==typeof E)try{if(y=n.props,m=(s=E.contextType)&&_[s.__c],g=s?m?m.props.value:s.__:_,e.__c?v=(c=n.__c=e.__c).__=c.__E:("prototype"in E&&E.prototype.render?n.__c=c=new E(y,g):(n.__c=c=new I(y,g),c.constructor=E,c.render=lt),m&&m.sub(c),c.props=y,c.state||(c.state={}),c.context=g,c.__n=_,h=c.__d=!0,c.__h=[],c._sb=[]),null==c.__s&&(c.__s=c.state),null!=E.getDerivedStateFromProps&&(c.__s==c.state&&(c.__s=M({},c.__s)),M(c.__s,E.getDerivedStateFromProps(y,c.__s))),a=c.props,p=c.state,c.__v=n,h)null==E.getDerivedStateFromProps&&null!=c.componentWillMount&&c.componentWillMount(),null!=c.componentDidMount&&c.__h.push(c.componentDidMount);else{if(null==E.getDerivedStateFromProps&&y!==a&&null!=c.componentWillReceiveProps&&c.componentWillReceiveProps(y,g),!c.__e&&(null!=c.shouldComponentUpdate&&!1===c.shouldComponentUpdate(y,c.__s,g)||n.__v===e.__v)){for(n.__v!==e.__v&&(c.props=y,c.state=c.__s,c.__d=!1),n.__e=e.__e,n.__k=e.__k,n.__k.forEach((function(t){t&&(t.__=n)})),b=0;b<c._sb.length;b++)c.__h.push(c._sb[b]);c._sb=[],c.__h.length&&r.push(c);break t}null!=c.componentWillUpdate&&c.componentWillUpdate(y,c.__s,g),null!=c.componentDidUpdate&&c.__h.push((function(){c.componentDidUpdate(a,p,d)}))}if(c.context=g,c.props=y,c.__P=t,c.__e=!1,k=C.__r,S=0,"prototype"in E&&E.prototype.render){for(c.state=c.__s,c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),w=0;w<c._sb.length;w++)c.__h.push(c._sb[w]);c._sb=[]}else do{c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),c.state=c.__s}while(c.__d&&++S<25);c.state=c.__s,null!=c.getChildContext&&(_=M(M({},_),c.getChildContext())),h||null==c.getSnapshotBeforeUpdate||(d=c.getSnapshotBeforeUpdate(a,p)),K(t,F(x=null!=s&&s.type===j&&null==s.key?s.props.children:s)?x:[x],n,e,_,i,o,r,u,f,l),c.base=n.__e,n.__u&=-161,c.__h.length&&r.push(c),v&&(c.__E=c.__=null)}catch(t){n.__v=null,f||null!=o?(n.__e=u,n.__u|=f?160:32,o[o.indexOf(u)]=null):(n.__e=e.__e,n.__k=e.__k),C.__e(t,n,e)}else null==o&&n.__v===e.__v?(n.__k=e.__k,n.__e=e.__e):n.__e=rt(e.__e,n,e,_,i,o,r,f,l);(s=C.diffed)&&s(n)}function ot(t,n,e){for(var _=0;_<e.length;_++)ut(e[_],e[++_],e[++_]);C.__c&&C.__c(n,t),t.some((function(n){try{t=n.__h,n.__h=[],t.some((function(t){t.call(n)}))}catch(t){C.__e(t,n.__v)}}))}function rt(t,n,e,_,i,o,r,u,f){var l,s,c,h,a,p,d,v=e.props,y=n.props,m=n.type;if("svg"===m&&(i=!0),null!=o)for(l=0;l<o.length;l++)if((a=o[l])&&"setAttribute"in a==!!m&&(m?a.localName===m:3===a.nodeType)){t=a,o[l]=null;break}if(null==t){if(null===m)return document.createTextNode(y);t=i?document.createElementNS("http://www.w3.org/2000/svg",m):document.createElement(m,y.is&&y),o=null,u=!1}if(null===m)v===y||u&&t.data===y||(t.data=y);else{if(o=o&&x.call(t.childNodes),v=e.props||T,!u&&null!=o)for(v={},l=0;l<t.attributes.length;l++)v[(a=t.attributes[l]).name]=a.value;for(l in v)a=v[l],"children"==l||("dangerouslySetInnerHTML"==l?c=a:"key"===l||l in y||nt(t,l,null,a,i));for(l in y)a=y[l],"children"==l?h=a:"dangerouslySetInnerHTML"==l?s=a:"value"==l?p=a:"checked"==l?d=a:"key"===l||u&&"function"!=typeof a||v[l]===a||nt(t,l,a,v[l],i);if(s)u||c&&(s.__html===c.__html||s.__html===t.innerHTML)||(t.innerHTML=s.__html),n.__k=[];else if(c&&(t.innerHTML=""),K(t,F(h)?h:[h],n,e,_,i&&"foreignObject"!==m,o,r,o?o[0]:e.__k&&q(e,0),u,f),null!=o)for(l=o.length;l--;)null!=o[l]&&W(o[l]);u||(l="value",void 0!==p&&(p!==t[l]||"progress"===m&&!p||"option"===m&&p!==v[l])&&nt(t,l,p,v[l],!1),l="checked",void 0!==d&&d!==t[l]&&nt(t,l,d,v[l],!1))}return t}function ut(t,n,e){try{"function"==typeof t?t(n):t.current=n}catch(t){C.__e(t,e)}}function ft(t,n,e){var _,i;if(C.unmount&&C.unmount(t),(_=t.ref)&&(_.current&&_.current!==t.__e||ut(_,null,n)),null!=(_=t.__c)){if(_.componentWillUnmount)try{_.componentWillUnmount()}catch(t){C.__e(t,n)}_.base=_.__P=null,t.__c=void 0}if(_=t.__k)for(i=0;i<_.length;i++)_[i]&&ft(_[i],n,e||"function"!=typeof t.type);e||null==t.__e||W(t.__e),t.__=t.__e=t.__d=void 0}function lt(t,n,e){return this.constructor(t,e)}function st(t,n,e){var _,i,o,r;C.__&&C.__(t,n),i=(_="function"==typeof e)?null:e&&e.__k||n.__k,o=[],r=[],it(n,t=(!_&&e||n).__k=L(j,null,[t]),i||T,T,void 0!==n.ownerSVGElement,!_&&e?[e]:i?null:n.firstChild?x.call(n.childNodes):null,o,!_&&e?e:i?i.__e:n.firstChild,_,r),t.__d=void 0,ot(o,t,r)}function ct(t,n){st(t,n,ct)}function ht(t,n,e){var _,i,o,r,u=M({},t.props);for(o in t.type&&t.type.defaultProps&&(r=t.type.defaultProps),n)"key"==o?_=n[o]:"ref"==o?i=n[o]:u[o]=void 0===n[o]&&void 0!==r?r[o]:n[o];return arguments.length>2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,_||t.key,i||t.ref,null)}function at(t,n){var e={__c:n="__cC"+D++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,_;return this.getChildContext||(e=[],(_={})[n]=this,this.getChildContext=function(){return _},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,z(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=V.slice,C={__e:function(t,n,e,_){for(var i,o,r;n=n.__;)if((i=n.__c)&&!i.__)try{if((o=i.constructor)&&null!=o.getDerivedStateFromError&&(i.setState(o.getDerivedStateFromError(t)),r=i.__d),null!=i.componentDidCatch&&(i.componentDidCatch(t,_||{}),r=i.__d),r)return i.__E=i}catch(n){t=n}throw t}},E=0,U=function(t){return null!=t&&null==t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=M({},this.state),"function"==typeof t&&(t=t(M({},e),this.props)),t&&M(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),z(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),z(this))},I.prototype.render=j,H=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,$=function(t,n){return t.__v.__b-n.__v.__b},J.__r=0,D=0;var pt,dt,vt,yt,mt=0,gt=[],bt=[],kt=C,St=kt.__b,wt=kt.__r,xt=kt.diffed,Ct=kt.__c,Et=kt.unmount,Ut=kt.__;function Ht(t,n){kt.__h&&kt.__h(dt,t,mt||n),mt=0;var e=dt.__H||(dt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:bt}),e.__[t]}function Pt(t){return mt=1,Nt(zt,t)}function Nt(t,n,e){var _=Ht(pt++,2);if(_.t=t,!_.__c&&(_.__=[e?e(n):zt(void 0,n),function(t){var n=_.__N?_.__N[0]:_.__[0],e=_.t(n,t);n!==e&&(_.__N=[e,_.__[1]],_.__c.setState({}))}],_.__c=dt,!dt.u)){var i=function(t,n,e){if(!_.__c.__H)return!0;var i=_.__c.__H.__.filter((function(t){return!!t.__c}));if(i.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return i.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&_.__c.props===t)&&(!o||o.call(this,t,n,e))};dt.u=!0;var o=dt.shouldComponentUpdate,r=dt.componentWillUpdate;dt.componentWillUpdate=function(t,n,e){if(this.__e){var _=o;o=void 0,i(t,n,e),o=_}r&&r.call(this,t,n,e)},dt.shouldComponentUpdate=i}return _.__N||_.__}function $t(t,n){var e=Ht(pt++,3);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__H.__h.push(e))}function Dt(t,n){var e=Ht(pt++,4);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__h.push(e))}function Tt(t){return mt=5,At((function(){return{current:t}}),[])}function Vt(t,n,e){mt=6,Dt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function At(t,n){var e=Ht(pt++,7);return Gt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Ft(t,n){return mt=8,At((function(){return t}),n)}function Mt(t){var n=dt.context[t.__c],e=Ht(pt++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(dt)),n.props.value):t.__}function Wt(t,n){kt.useDebugValue&&kt.useDebugValue(n?n(t):t)}function Lt(t){var n=Ht(pt++,10),e=Pt();return n.__=t,dt.componentDidCatch||(dt.componentDidCatch=function(t,_){n.__&&n.__(t,_),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Ot(){var t=Ht(pt++,11);if(!t.__){for(var n=dt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Rt(){for(var t;t=gt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(qt),t.__H.__h.forEach(Bt),t.__H.__h=[]}catch(n){t.__H.__h=[],kt.__e(n,t.__v)}}kt.__b=function(t){dt=null,St&&St(t)},kt.__=function(t,n){t&&n.__k&&n.__k.__m&&(t.__m=n.__k.__m),Ut&&Ut(t,n)},kt.__r=function(t){wt&&wt(t),pt=0;var n=(dt=t.__c).__H;n&&(vt===dt?(n.__h=[],dt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=bt,t.__N=t.i=void 0}))):(n.__h.forEach(qt),n.__h.forEach(Bt),n.__h=[],pt=0)),vt=dt},kt.diffed=function(t){xt&&xt(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==gt.push(n)&&yt===kt.requestAnimationFrame||((yt=kt.requestAnimationFrame)||It)(Rt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==bt&&(t.__=t.__V),t.i=void 0,t.__V=bt}))),vt=dt=null},kt.__c=function(t,n){n.some((function(t){try{t.__h.forEach(qt),t.__h=t.__h.filter((function(t){return!t.__||Bt(t)}))}catch(u){n.some((function(t){t.__h&&(t.__h=[])})),n=[],kt.__e(u,t.__v)}})),Ct&&Ct(t,n)},kt.unmount=function(t){Et&&Et(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{qt(t)}catch(t){n=t}})),e.__H=void 0,n&&kt.__e(n,e.__v))};var jt="function"==typeof requestAnimationFrame;function It(t){var n,e=function(){clearTimeout(_),jt&&cancelAnimationFrame(n),setTimeout(t)},_=setTimeout(e,100);jt&&(n=requestAnimationFrame(e))}function qt(t){var n=dt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),dt=n}function Bt(t){var n=dt;t.__c=t.__(),dt=n}function Gt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function zt(t,n){return"function"==typeof n?n(t):n}function Jt(t,n){C[t]=n.bind(null,C[t]||(()=>{}))}let Kt,Qt;function Xt(t){if(Qt)Qt();Qt=t&&t.S()}function Yt({data:t}){const n=tn(t);n.value=t;const e=At(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!U(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return m(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Yt.displayName="_st";Object.defineProperties(h.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Yt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});Jt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let _ in e){if("children"===_)continue;let i=e[_];if(i instanceof h){if(!t)n.__np=t={};t[_]=i;e[_]=i.peek()}}}t(n)});Jt("__r",(t,n)=>{Xt();let e,_=n.__c;if(_){_.__$f&=-2;e=_.__$u;if(void 0===e)_.__$u=e=function(t){let n;w((function(){n=this}));n.c=()=>{_.__$f|=1;_.setState({})};return n}()}Kt=_;Xt(e);t(n)});Jt("__e",(t,n,e,_)=>{Xt();Kt=void 0;t(n,e,_)});Jt("diffed",(t,n)=>{Xt();Kt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,_=n.props;if(t){let n=e.U;if(n)for(let e in n){let _=n[e];if(void 0!==_&&!(e in t)){_.d();n[e]=void 0}}else{n={};e.U=n}for(let i in t){let o=n[i],r=t[i];if(void 0===o){o=Zt(e,i,r,_);n[i]=o}else o.o(r,_)}}}t(n)});function Zt(t,n,e,_){const i=n in t&&void 0===t.ownerSVGElement,o=a(e);return{o:(t,n)=>{o.value=t;_=n},d:w(()=>{const e=o.value.value;if(_[n]!==e){_[n]=e;if(i)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}Jt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});Jt("__h",(t,n,e,_)=>{if(_<3||9===_)n.__$f|=2;t(n,e,_)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let _ in n)return!0;for(let _ in t)if("__source"!==_&&t[_]!==this.props[_])return!0;for(let _ in this.props)if(!(_ in t))return!0;return!1};function tn(t){return At(()=>a(t),[])}function nn(t){const n=Tt(t);n.current=t;Kt.__$f|=4;return At(()=>m(()=>n.current()),[])}function en(t){const n=Tt(t);n.current=t;$t(()=>w(()=>n.current()),[])}var _n=function(t,n,e,_){var i;n[0]=0;for(var o=1;o<n.length;o++){var r=n[o++],u=n[o]?(n[0]|=r?1:2,e[n[o++]]):n[++o];3===r?_[0]=u:4===r?_[1]=Object.assign(_[1]||{},u):5===r?(_[1]=_[1]||{})[n[++o]]=u:6===r?_[1][n[++o]]+=u+"":r?(i=t.apply(u,_n(t,u,e,["",null])),_.push(i),u[0]?n[0]|=2:(n[o-2]=0,n[o]=i)):_.push(u)}return _},on=new Map;function rn(t){var n=on.get(this);return n||(n=new Map,on.set(this,n)),(n=_n(this,n.get(t)||(n.set(t,n=function(t){for(var n,e,_=1,i="",o="",r=[0],u=function(t){1===_&&(t||(i=i.replace(/^\s*\n\s*|\s*\n\s*$/g,"")))?r.push(0,t,i):3===_&&(t||i)?(r.push(3,t,i),_=2):2===_&&"..."===i&&t?r.push(4,t,0):2===_&&i&&!t?r.push(5,0,!0,i):_>=5&&((i||!t&&5===_)&&(r.push(_,0,i,e),_=6),t&&(r.push(_,t,0,e),_=6)),i=""},f=0;f<t.length;f++){f&&(1===_&&u(),u(f));for(var l=0;l<t[f].length;l++)n=t[f][l],1===_?"<"===n?(u(),r=[r],_=3):i+=n:4===_?"--"===i&&">"===n?(_=1,i=""):i=n+i[0]:o?n===o?o="":i+=n:'"'===n||"'"===n?o=n:">"===n?(u(),_=1):_&&("="===n?(_=5,e=i,i=""):"/"===n&&(_<5||">"===t[f][l+1])?(u(),3===_&&(r=r[0]),_=r,(r=r[0]).push(2,0,_),_=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),_=2):i+=n),3===_&&"!--"===i&&(_=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var un=rn.bind(L);export{I as Component,j as Fragment,h as Signal,_ as batch,ht as cloneElement,m as computed,at as createContext,L as createElement,R as createRef,w as effect,L as h,un as html,ct as hydrate,U as isValidElement,C as options,st as render,a as signal,Y as toChildArray,u as untracked,Ft as useCallback,nn as useComputed,Mt as useContext,Wt as useDebugValue,$t as useEffect,Lt as useErrorBoundary,Ot as useId,Vt as useImperativeHandle,Dt as useLayoutEffect,At as useMemo,Nt as useReducer,Tt as useRef,tn as useSignal,en as useSignalEffect,Pt as useState};
+function t(){throw new Error("Cycle detected")}function n(){if(u>1){u--;return}let t,n=!1;while(void 0!==_){let i=_;_=void 0;f++;while(void 0!==i){const _=i.o;i.o=void 0;i.f&=-3;if(!(8&i.f)&&a(i))try{i.c()}catch(e){if(!n){t=e;n=!0}}i=_}}f=0;u--;if(n)throw t}function e(t){if(u>0)return t();u++;try{return t()}finally{n()}}let i,_,o=0;function r(t){if(o>0)return t();const n=i;i=void 0;o++;try{return t()}finally{o--;i=n}}let u=0,f=0,l=0;function s(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function c(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}c.prototype.h=function(){return!0};c.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};c.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};c.prototype.subscribe=function(t){const n=this;return S((function(){const e=n.value,i=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=i}}))};c.prototype.valueOf=function(){return this.value};c.prototype.toString=function(){return this.value+""};c.prototype.toJSON=function(){return this.value};c.prototype.peek=function(){return this.v};Object.defineProperty(c.prototype,"value",{get(){const t=s(this);if(void 0!==t)t.i=this.i;return this.v},set(e){if(i instanceof v)!function(){throw new Error("Computed cannot have side-effects")}();if(e!==this.v){if(f>100)t();this.v=e;this.i++;l++;u++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{n()}}}});function h(t){return new c(t)}function a(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function p(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function d(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function v(t){c.call(this,void 0);this.x=t;this.s=void 0;this.g=l-1;this.f=4}(v.prototype=new c).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===l)return!0;this.g=l;this.f|=1;if(this.i>0&&!a(this)){this.f&=-2;return!0}const t=i;try{p(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;d(this);this.f&=-2;return!0};v.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}c.prototype.S.call(this,t)};v.prototype.U=function(t){if(void 0!==this.t){c.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};v.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};v.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(v.prototype,"value",{get(){if(1&this.f)t();const n=s(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function y(t){return new v(t)}function m(t){const e=t.u;t.u=void 0;if("function"==typeof e){u++;const _=i;i=void 0;try{e()}catch(n){t.f&=-2;t.f|=8;g(t);throw n}finally{i=_;n()}}}function g(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;m(t)}function b(t){if(i!==this)throw new Error("Out-of-order effect");d(this);i=t;this.f&=-2;if(8&this.f)g(this);n()}function k(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}k.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};k.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;m(this);p(this);u++;const n=i;i=this;return b.bind(this,n)};k.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=_;_=this}};k.prototype.d=function(){this.f|=8;if(!(1&this.f))g(this)};function S(t){const n=new k(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,w,C,E,U,H,N,P,$,D={},T=[],V=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,A=Array.isArray;function F(t,n){for(var e in n)t[e]=n[e];return t}function M(t){var n=t.parentNode;n&&n.removeChild(t)}function W(t,n,e){var i,_,o,r={};for(o in n)"key"==o?i=n[o]:"ref"==o?_=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,i,_,null)}function O(t,n,e,i,_){var o={type:t,props:n,key:e,ref:i,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,__h:null,constructor:void 0,__v:null==_?++C:_};return null==_&&null!=w.vnode&&w.vnode(o),o}function L(){return{current:null}}function R(t){return t.children}function I(t,n){this.props=t,this.context=n}function j(t,n){if(null==n)return t.__?j(t.__,t.__.__k.indexOf(t)+1):null;for(var e;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e)return e.__e;return"function"==typeof t.type?j(t):null}function B(t){var n,e;if(null!=(t=t.__)&&null!=t.__c){for(t.__e=t.__c.base=null,n=0;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e){t.__e=t.__c.base=e.__e;break}return B(t)}}function q(t){(!t.__d&&(t.__d=!0)&&U.push(t)&&!G.__r++||H!==w.debounceRendering)&&((H=w.debounceRendering)||N)(G)}function G(){var t,n,e,i,_,o,r,u,f;for(U.sort(P);t=U.shift();)t.__d&&(n=U.length,i=void 0,_=void 0,o=void 0,u=(r=(e=t).__v).__e,(f=e.__P)&&(i=[],_=[],(o=F({},r)).__v=r.__v+1,it(f,r,o,e.__n,void 0!==f.ownerSVGElement,null!=r.__h?[u]:null,i,null==u?j(r):u,r.__h,_),_t(i,r,_),r.__e!=u&&B(r)),U.length>n&&U.sort(P));G.__r=0}function z(t,n,e,i,_,o,r,u,f,l,s){var c,h,a,p,d,v,y,m,g,b,k=0,S=i&&i.__k||T,x=S.length,w=x,C=n.length;for(e.__k=[],c=0;c<C;c++)null!=(p=e.__k[c]=null==(p=n[c])||"boolean"==typeof p||"function"==typeof p?null:"string"==typeof p||"number"==typeof p||"bigint"==typeof p?O(null,p,null,null,p):A(p)?O(R,{children:p},null,null,null):p.__b>0?O(p.type,p.props,p.key,p.ref?p.ref:null,p.__v):p)&&(p.__=e,p.__b=e.__b+1,-1===(m=X(p,S,y=c+k,w))?a=D:(a=S[m]||D,S[m]=void 0,w--),it(t,p,a,_,o,r,u,f,l,s),d=p.__e,(h=p.ref)&&a.ref!=h&&(a.ref&&rt(a.ref,null,p),s.push(h,p.__c||d,p)),null!=d&&(null==v&&(v=d),b=!(g=a===D||null===a.__v)&&m===y,g?-1==m&&k--:m!==y&&(m===y+1?(k++,b=!0):m>y?w>C-y?(k+=m-y,b=!0):k--:k=m<y&&m==y-1?m-y:0),y=c+k,b=b||m==c&&!g,"function"!=typeof p.type||m===y&&a.__k!==p.__k?"function"==typeof p.type||b?void 0!==p.__d?(f=p.__d,p.__d=void 0):f=d.nextSibling:f=Q(t,d,f):f=J(p,f,t),"function"==typeof e.type&&(e.__d=f)));for(e.__e=v,c=x;c--;)null!=S[c]&&("function"==typeof e.type&&null!=S[c].__e&&S[c].__e==e.__d&&(e.__d=S[c].__e.nextSibling),ut(S[c],S[c]))}function J(t,n,e){for(var i,_=t.__k,o=0;_&&o<_.length;o++)(i=_[o])&&(i.__=t,n="function"==typeof i.type?J(i,n,e):Q(e,i.__e,n));return n}function K(t,n){return n=n||[],null==t||"boolean"==typeof t||(A(t)?t.some((function(t){K(t,n)})):n.push(t)),n}function Q(t,n,e){return null==e||e.parentNode!==t?t.insertBefore(n,null):n==e&&null!=n.parentNode||t.insertBefore(n,e),n.nextSibling}function X(t,n,e,i){var _=t.key,o=t.type,r=e-1,u=e+1,f=n[e];if(null===f||f&&_==f.key&&o===f.type)return e;if(i>(null!=f?1:0))for(;r>=0||u<n.length;){if(r>=0){if((f=n[r])&&_==f.key&&o===f.type)return r;r--}if(u<n.length){if((f=n[u])&&_==f.key&&o===f.type)return u;u++}}return-1}function Y(t,n,e,i,_){var o;for(o in e)"children"===o||"key"===o||o in n||tt(t,o,null,e[o],i);for(o in n)_&&"function"!=typeof n[o]||"children"===o||"key"===o||"value"===o||"checked"===o||e[o]===n[o]||tt(t,o,n[o],e[o],i)}function Z(t,n,e){"-"===n[0]?t.setProperty(n,null==e?"":e):t[n]=null==e?"":"number"!=typeof e||V.test(n)?e:e+"px"}function tt(t,n,e,i,_){var o;t:if("style"===n)if("string"==typeof e)t.style.cssText=e;else{if("string"==typeof i&&(t.style.cssText=i=""),i)for(n in i)e&&n in e||Z(t.style,n,"");if(e)for(n in e)i&&e[n]===i[n]||Z(t.style,n,e[n])}else if("o"===n[0]&&"n"===n[1])o=n!==(n=n.replace(/Capture$/,"")),n=n.toLowerCase()in t?n.toLowerCase().slice(2):n.slice(2),t.l||(t.l={}),t.l[n+o]=e,e?i||t.addEventListener(n,o?et:nt,o):t.removeEventListener(n,o?et:nt,o);else if("dangerouslySetInnerHTML"!==n){if(_)n=n.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if("width"!==n&&"height"!==n&&"href"!==n&&"list"!==n&&"form"!==n&&"tabIndex"!==n&&"download"!==n&&"rowSpan"!==n&&"colSpan"!==n&&n in t)try{t[n]=null==e?"":e;break t}catch(t){}"function"==typeof e||(null==e||!1===e&&"-"!==n[4]?t.removeAttribute(n):t.setAttribute(n,e))}}function nt(t){return this.l[t.type+!1](w.event?w.event(t):t)}function et(t){return this.l[t.type+!0](w.event?w.event(t):t)}function it(t,n,e,i,_,o,r,u,f,l){var s,c,h,a,p,d,v,y,m,g,b,k,S,x,C,E=n.type;if(void 0!==n.constructor)return null;null!=e.__h&&(f=e.__h,u=n.__e=e.__e,n.__h=null,o=[u]),(s=w.__b)&&s(n);try{t:if("function"==typeof E){if(y=n.props,m=(s=E.contextType)&&i[s.__c],g=s?m?m.props.value:s.__:i,e.__c?v=(c=n.__c=e.__c).__=c.__E:("prototype"in E&&E.prototype.render?n.__c=c=new E(y,g):(n.__c=c=new I(y,g),c.constructor=E,c.render=ft),m&&m.sub(c),c.props=y,c.state||(c.state={}),c.context=g,c.__n=i,h=c.__d=!0,c.__h=[],c._sb=[]),null==c.__s&&(c.__s=c.state),null!=E.getDerivedStateFromProps&&(c.__s==c.state&&(c.__s=F({},c.__s)),F(c.__s,E.getDerivedStateFromProps(y,c.__s))),a=c.props,p=c.state,c.__v=n,h)null==E.getDerivedStateFromProps&&null!=c.componentWillMount&&c.componentWillMount(),null!=c.componentDidMount&&c.__h.push(c.componentDidMount);else{if(null==E.getDerivedStateFromProps&&y!==a&&null!=c.componentWillReceiveProps&&c.componentWillReceiveProps(y,g),!c.__e&&(null!=c.shouldComponentUpdate&&!1===c.shouldComponentUpdate(y,c.__s,g)||n.__v===e.__v)){for(n.__v!==e.__v&&(c.props=y,c.state=c.__s,c.__d=!1),n.__e=e.__e,n.__k=e.__k,n.__k.forEach((function(t){t&&(t.__=n)})),b=0;b<c._sb.length;b++)c.__h.push(c._sb[b]);c._sb=[],c.__h.length&&r.push(c);break t}null!=c.componentWillUpdate&&c.componentWillUpdate(y,c.__s,g),null!=c.componentDidUpdate&&c.__h.push((function(){c.componentDidUpdate(a,p,d)}))}if(c.context=g,c.props=y,c.__P=t,c.__e=!1,k=w.__r,S=0,"prototype"in E&&E.prototype.render){for(c.state=c.__s,c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),x=0;x<c._sb.length;x++)c.__h.push(c._sb[x]);c._sb=[]}else do{c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),c.state=c.__s}while(c.__d&&++S<25);c.state=c.__s,null!=c.getChildContext&&(i=F(F({},i),c.getChildContext())),h||null==c.getSnapshotBeforeUpdate||(d=c.getSnapshotBeforeUpdate(a,p)),z(t,A(C=null!=s&&s.type===R&&null==s.key?s.props.children:s)?C:[C],n,e,i,_,o,r,u,f,l),c.base=n.__e,n.__h=null,c.__h.length&&r.push(c),v&&(c.__E=c.__=null)}else null==o&&n.__v===e.__v?(n.__k=e.__k,n.__e=e.__e):n.__e=ot(e.__e,n,e,i,_,o,r,f,l);(s=w.diffed)&&s(n)}catch(t){n.__v=null,(f||null!=o)&&(n.__e=u,n.__h=!!f,o[o.indexOf(u)]=null),w.__e(t,n,e)}}function _t(t,n,e){for(var i=0;i<e.length;i++)rt(e[i],e[++i],e[++i]);w.__c&&w.__c(n,t),t.some((function(n){try{t=n.__h,n.__h=[],t.some((function(t){t.call(n)}))}catch(t){w.__e(t,n.__v)}}))}function ot(t,n,e,i,_,o,r,u,f){var l,s,c,h=e.props,a=n.props,p=n.type,d=0;if("svg"===p&&(_=!0),null!=o)for(;d<o.length;d++)if((l=o[d])&&"setAttribute"in l==!!p&&(p?l.localName===p:3===l.nodeType)){t=l,o[d]=null;break}if(null==t){if(null===p)return document.createTextNode(a);t=_?document.createElementNS("http://www.w3.org/2000/svg",p):document.createElement(p,a.is&&a),o=null,u=!1}if(null===p)h===a||u&&t.data===a||(t.data=a);else{if(o=o&&x.call(t.childNodes),s=(h=e.props||D).dangerouslySetInnerHTML,c=a.dangerouslySetInnerHTML,!u){if(null!=o)for(h={},d=0;d<t.attributes.length;d++)h[t.attributes[d].name]=t.attributes[d].value;(c||s)&&(c&&(s&&c.__html==s.__html||c.__html===t.innerHTML)||(t.innerHTML=c&&c.__html||""))}if(Y(t,a,h,_,u),c)n.__k=[];else if(z(t,A(d=n.props.children)?d:[d],n,e,i,_&&"foreignObject"!==p,o,r,o?o[0]:e.__k&&j(e,0),u,f),null!=o)for(d=o.length;d--;)null!=o[d]&&M(o[d]);u||("value"in a&&void 0!==(d=a.value)&&(d!==t.value||"progress"===p&&!d||"option"===p&&d!==h.value)&&tt(t,"value",d,h.value,!1),"checked"in a&&void 0!==(d=a.checked)&&d!==t.checked&&tt(t,"checked",d,h.checked,!1))}return t}function rt(t,n,e){try{"function"==typeof t?t(n):t.current=n}catch(t){w.__e(t,e)}}function ut(t,n,e){var i,_;if(w.unmount&&w.unmount(t),(i=t.ref)&&(i.current&&i.current!==t.__e||rt(i,null,n)),null!=(i=t.__c)){if(i.componentWillUnmount)try{i.componentWillUnmount()}catch(t){w.__e(t,n)}i.base=i.__P=null,t.__c=void 0}if(i=t.__k)for(_=0;_<i.length;_++)i[_]&&ut(i[_],n,e||"function"!=typeof t.type);e||null==t.__e||M(t.__e),t.__=t.__e=t.__d=void 0}function ft(t,n,e){return this.constructor(t,e)}function lt(t,n,e){var i,_,o,r;w.__&&w.__(t,n),_=(i="function"==typeof e)?null:e&&e.__k||n.__k,o=[],r=[],it(n,t=(!i&&e||n).__k=W(R,null,[t]),_||D,D,void 0!==n.ownerSVGElement,!i&&e?[e]:_?null:n.firstChild?x.call(n.childNodes):null,o,!i&&e?e:_?_.__e:n.firstChild,i,r),_t(o,t,r)}function st(t,n){lt(t,n,st)}function ct(t,n,e){var i,_,o,r,u=F({},t.props);for(o in t.type&&t.type.defaultProps&&(r=t.type.defaultProps),n)"key"==o?i=n[o]:"ref"==o?_=n[o]:u[o]=void 0===n[o]&&void 0!==r?r[o]:n[o];return arguments.length>2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,i||t.key,_||t.ref,null)}function ht(t,n){var e={__c:n="__cC"+$++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,i;return this.getChildContext||(e=[],(i={})[n]=this,this.getChildContext=function(){return i},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,q(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=T.slice,w={__e:function(t,n,e,i){for(var _,o,r;n=n.__;)if((_=n.__c)&&!_.__)try{if((o=_.constructor)&&null!=o.getDerivedStateFromError&&(_.setState(o.getDerivedStateFromError(t)),r=_.__d),null!=_.componentDidCatch&&(_.componentDidCatch(t,i||{}),r=_.__d),r)return _.__E=_}catch(n){t=n}throw t}},C=0,E=function(t){return null!=t&&void 0===t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=F({},this.state),"function"==typeof t&&(t=t(F({},e),this.props)),t&&F(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),q(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),q(this))},I.prototype.render=R,U=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,P=function(t,n){return t.__v.__b-n.__v.__b},G.__r=0,$=0;var at,pt,dt,vt,yt=0,mt=[],gt=[],bt=w.__b,kt=w.__r,St=w.diffed,xt=w.__c,wt=w.unmount;function Ct(t,n){w.__h&&w.__h(pt,t,yt||n),yt=0;var e=pt.__H||(pt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:gt}),e.__[t]}function Et(t){return yt=1,Ut(Bt,t)}function Ut(t,n,e){var i=Ct(at++,2);if(i.t=t,!i.__c&&(i.__=[e?e(n):Bt(void 0,n),function(t){var n=i.__N?i.__N[0]:i.__[0],e=i.t(n,t);n!==e&&(i.__N=[e,i.__[1]],i.__c.setState({}))}],i.__c=pt,!pt.u)){var _=function(t,n,e){if(!i.__c.__H)return!0;var _=i.__c.__H.__.filter((function(t){return t.__c}));if(_.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return _.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&i.__c.props===t)&&(!o||o.call(this,t,n,e))};pt.u=!0;var o=pt.shouldComponentUpdate,r=pt.componentWillUpdate;pt.componentWillUpdate=function(t,n,e){if(this.__e){var i=o;o=void 0,_(t,n,e),o=i}r&&r.call(this,t,n,e)},pt.shouldComponentUpdate=_}return i.__N||i.__}function Ht(t,n){var e=Ct(at++,3);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__H.__h.push(e))}function Nt(t,n){var e=Ct(at++,4);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__h.push(e))}function Pt(t){return yt=5,Dt((function(){return{current:t}}),[])}function $t(t,n,e){yt=6,Nt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function Dt(t,n){var e=Ct(at++,7);return jt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Tt(t,n){return yt=8,Dt((function(){return t}),n)}function Vt(t){var n=pt.context[t.__c],e=Ct(at++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(pt)),n.props.value):t.__}function At(t,n){w.useDebugValue&&w.useDebugValue(n?n(t):t)}function Ft(t){var n=Ct(at++,10),e=Et();return n.__=t,pt.componentDidCatch||(pt.componentDidCatch=function(t,i){n.__&&n.__(t,i),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Mt(){var t=Ct(at++,11);if(!t.__){for(var n=pt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Wt(){for(var t;t=mt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(Rt),t.__H.__h.forEach(It),t.__H.__h=[]}catch(u){t.__H.__h=[],w.__e(u,t.__v)}}w.__b=function(t){pt=null,bt&&bt(t)},w.__r=function(t){kt&&kt(t),at=0;var n=(pt=t.__c).__H;n&&(dt===pt?(n.__h=[],pt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=gt,t.__N=t.i=void 0}))):(n.__h.forEach(Rt),n.__h.forEach(It),n.__h=[],at=0)),dt=pt},w.diffed=function(t){St&&St(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==mt.push(n)&&vt===w.requestAnimationFrame||((vt=w.requestAnimationFrame)||Lt)(Wt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==gt&&(t.__=t.__V),t.i=void 0,t.__V=gt}))),dt=pt=null},w.__c=function(t,n){n.some((function(t){try{t.__h.forEach(Rt),t.__h=t.__h.filter((function(t){return!t.__||It(t)}))}catch(s){n.some((function(t){t.__h&&(t.__h=[])})),n=[],w.__e(s,t.__v)}})),xt&&xt(t,n)},w.unmount=function(t){wt&&wt(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{Rt(t)}catch(t){n=t}})),e.__H=void 0,n&&w.__e(n,e.__v))};var Ot="function"==typeof requestAnimationFrame;function Lt(t){var n,e=function(){clearTimeout(i),Ot&&cancelAnimationFrame(n),setTimeout(t)},i=setTimeout(e,100);Ot&&(n=requestAnimationFrame(e))}function Rt(t){var n=pt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),pt=n}function It(t){var n=pt;t.__c=t.__(),pt=n}function jt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function Bt(t,n){return"function"==typeof n?n(t):n}function qt(t,n){w[t]=n.bind(null,w[t]||(()=>{}))}let Gt,zt;function Jt(t){if(zt)zt();zt=t&&t.S()}function Kt({data:t}){const n=Xt(t);n.value=t;const e=Dt(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!E(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return y(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Kt.displayName="_st";Object.defineProperties(c.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Kt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});qt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let i in e){if("children"===i)continue;let _=e[i];if(_ instanceof c){if(!t)n.__np=t={};t[i]=_;e[i]=_.peek()}}}t(n)});qt("__r",(t,n)=>{Jt();let e,i=n.__c;if(i){i.__$f&=-2;e=i.__$u;if(void 0===e)i.__$u=e=function(t){let n;S((function(){n=this}));n.c=()=>{i.__$f|=1;i.setState({})};return n}()}Gt=i;Jt(e);t(n)});qt("__e",(t,n,e,i)=>{Jt();Gt=void 0;t(n,e,i)});qt("diffed",(t,n)=>{Jt();Gt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,i=n.props;if(t){let n=e.U;if(n)for(let e in n){let i=n[e];if(void 0!==i&&!(e in t)){i.d();n[e]=void 0}}else{n={};e.U=n}for(let _ in t){let o=n[_],r=t[_];if(void 0===o){o=Qt(e,_,r,i);n[_]=o}else o.o(r,i)}}}t(n)});function Qt(t,n,e,i){const _=n in t&&void 0===t.ownerSVGElement,o=h(e);return{o:(t,n)=>{o.value=t;i=n},d:S(()=>{const e=o.value.value;if(i[n]!==e){i[n]=e;if(_)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}qt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});qt("__h",(t,n,e,i)=>{if(i<3||9===i)n.__$f|=2;t(n,e,i)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let i in n)return!0;for(let i in t)if("__source"!==i&&t[i]!==this.props[i])return!0;for(let i in this.props)if(!(i in t))return!0;return!1};function Xt(t){return Dt(()=>h(t),[])}function Yt(t){const n=Pt(t);n.current=t;Gt.__$f|=4;return Dt(()=>y(()=>n.current()),[])}function Zt(t){const n=Pt(t);n.current=t;Ht(()=>S(()=>n.current()),[])}var tn=function(t,n,e,i){var _;n[0]=0;for(var o=1;o<n.length;o++){var r=n[o++],u=n[o]?(n[0]|=r?1:2,e[n[o++]]):n[++o];3===r?i[0]=u:4===r?i[1]=Object.assign(i[1]||{},u):5===r?(i[1]=i[1]||{})[n[++o]]=u:6===r?i[1][n[++o]]+=u+"":r?(_=t.apply(u,tn(t,u,e,["",null])),i.push(_),u[0]?n[0]|=2:(n[o-2]=0,n[o]=_)):i.push(u)}return i},nn=new Map;function en(t){var n=nn.get(this);return n||(n=new Map,nn.set(this,n)),(n=tn(this,n.get(t)||(n.set(t,n=function(t){for(var n,e,i=1,_="",o="",r=[0],u=function(t){1===i&&(t||(_=_.replace(/^\s*\n\s*|\s*\n\s*$/g,"")))?r.push(0,t,_):3===i&&(t||_)?(r.push(3,t,_),i=2):2===i&&"..."===_&&t?r.push(4,t,0):2===i&&_&&!t?r.push(5,0,!0,_):i>=5&&((_||!t&&5===i)&&(r.push(i,0,_,e),i=6),t&&(r.push(i,t,0,e),i=6)),_=""},f=0;f<t.length;f++){f&&(1===i&&u(),u(f));for(var l=0;l<t[f].length;l++)n=t[f][l],1===i?"<"===n?(u(),r=[r],i=3):_+=n:4===i?"--"===_&&">"===n?(i=1,_=""):_=n+_[0]:o?n===o?o="":_+=n:'"'===n||"'"===n?o=n:">"===n?(u(),i=1):i&&("="===n?(i=5,e=_,_=""):"/"===n&&(i<5||">"===t[f][l+1])?(u(),3===i&&(r=r[0]),i=r,(r=r[0]).push(2,0,i),i=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),i=2):_+=n),3===i&&"!--"===_&&(i=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var _n=en.bind(W);export{I as Component,R as Fragment,c as Signal,e as batch,ct as cloneElement,y as computed,ht as createContext,W as createElement,L as createRef,S as effect,W as h,_n as html,st as hydrate,E as isValidElement,w as options,lt as render,h as signal,K as toChildArray,r as untracked,Tt as useCallback,Yt as useComputed,Vt as useContext,At as useDebugValue,Ht as useEffect,Ft as useErrorBoundary,Mt as useId,$t as useImperativeHandle,Nt as useLayoutEffect,Dt as useMemo,Ut as useReducer,Pt as useRef,Xt as useSignal,Zt as useSignalEffect,Et as useState};
diff --git a/llama.cpp/server/public/json-schema-to-grammar.mjs b/llama.cpp/server/public/json-schema-to-grammar.mjs
index baa6761c..3f1b255c 100644
--- a/llama.cpp/server/public/json-schema-to-grammar.mjs
+++ b/llama.cpp/server/public/json-schema-to-grammar.mjs
@@ -1,50 +1,25 @@
-// WARNING: This file was ported from json-schema-to-grammar.py, please fix bugs / add features there first.
const SPACE_RULE = '" "?';
const PRIMITIVE_RULES = {
boolean: '("true" | "false") space',
number: '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space',
integer: '("-"? ([0-9] | [1-9] [0-9]*)) space',
- value: 'object | array | string | number | boolean',
- object: '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space',
- array: '"[" space ( value ("," space value)* )? "]" space',
- uuid: '"\\"" ' + [8, 4, 4, 4, 12].map(n => [...new Array(n)].map(_ => '[0-9a-fA-F]').join('')).join(' "-" ') + ' "\\"" space',
string: ` "\\"" (
[^"\\\\] |
"\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
)* "\\"" space`,
null: '"null" space',
};
-const OBJECT_RULE_NAMES = ['object', 'array', 'string', 'number', 'boolean', 'null', 'value'];
-
-// TODO: support "uri", "email" string formats
-const DATE_RULES = {
- 'date' : '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )',
- 'time' : '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )',
- 'date-time': 'date "T" time',
- 'date-string': '"\\"" date "\\"" space',
- 'time-string': '"\\"" time "\\"" space',
- 'date-time-string': '"\\"" date-time "\\"" space',
-};
-
-const RESERVED_NAMES = {'root': true, ...PRIMITIVE_RULES, ...DATE_RULES};
const INVALID_RULE_CHARS_RE = /[^\dA-Za-z-]+/g;
const GRAMMAR_LITERAL_ESCAPE_RE = /[\n\r"]/g;
-const GRAMMAR_RANGE_LITERAL_ESCAPE_RE = /[\n\r"\]\-\\]/g;
-const GRAMMAR_LITERAL_ESCAPES = { '\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]' };
-
-const NON_LITERAL_SET = new Set('|.()[]{}*+?');
-const ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = new Set('[]()|{}*+?');
+const GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"'};
export class SchemaConverter {
- constructor(options) {
- this._propOrder = options.prop_order || {};
- this._allowFetch = options.allow_fetch || false;
- this._dotall = options.dotall || false;
- this._rules = {'space': SPACE_RULE};
- this._refs = {};
- this._refsBeingResolved = new Set();
+ constructor(propOrder) {
+ this._propOrder = propOrder || {};
+ this._rules = new Map();
+ this._rules.set('space', SPACE_RULE);
}
_formatLiteral(literal) {
@@ -55,490 +30,83 @@ export class SchemaConverter {
return `"${escaped}"`;
}
- _formatRangeChar(literal) {
- return JSON.stringify(literal).slice(1, -1).replace(
- GRAMMAR_RANGE_LITERAL_ESCAPE_RE,
- m => GRAMMAR_LITERAL_ESCAPES[m]
- );
- }
-
_addRule(name, rule) {
let escName = name.replace(INVALID_RULE_CHARS_RE, '-');
let key = escName;
- if (escName in this._rules) {
- if (this._rules[escName] === rule) {
+ if (this._rules.has(escName)) {
+ if (this._rules.get(escName) === rule) {
return key;
}
let i = 0;
- while ((`${escName}${i}` in this._rules) && (this._rules[`${escName}${i}`] !== rule)) {
+ while (this._rules.has(`${escName}${i}`)) {
i += 1;
}
key = `${escName}${i}`;
}
- this._rules[key] = rule;
+ this._rules.set(key, rule);
return key;
}
- async resolveRefs(schema, url) {
- const visit = async (n) => {
- if (Array.isArray(n)) {
- return Promise.all(n.map(visit));
- } else if (typeof n === 'object' && n !== null) {
- let ref = n.$ref;
- let target;
- if (ref !== undefined && !this._refs[ref]) {
- if (ref.startsWith('https://')) {
- if (!this._allowFetch) {
- throw new Error('Fetching remote schemas is not allowed (use --allow-fetch for force)');
- }
- const fetch = (await import('node-fetch')).default;
-
- const fragSplit = ref.split('#');
- const baseUrl = fragSplit[0];
-
- target = this._refs[baseUrl];
- if (!target) {
- target = await this.resolveRefs(await fetch(ref).then(res => res.json()), baseUrl);
- this._refs[baseUrl] = target;
- }
-
- if (fragSplit.length === 1 || fragSplit[fragSplit.length - 1] === '') {
- return target;
- }
- } else if (ref.startsWith('#/')) {
- target = schema;
- ref = `${url}${ref}`;
- n.$ref = ref;
- } else {
- throw new Error(`Unsupported ref ${ref}`);
- }
-
- const selectors = ref.split('#')[1].split('/').slice(1);
- for (const sel of selectors) {
- if (!target || !(sel in target)) {
- throw new Error(`Error resolving ref ${ref}: ${sel} not in ${JSON.stringify(target)}`);
- }
- target = target[sel];
- }
-
- this._refs[ref] = target;
- } else {
- await Promise.all(Object.values(n).map(visit));
- }
- }
-
- return n;
- };
-
- return visit(schema);
- }
-
- _generateUnionRule(name, altSchemas) {
- return altSchemas
- .map((altSchema, i) => this.visit(altSchema, `${name ?? ''}${name ? '-' : 'alternative-'}${i}`))
- .join(' | ');
- }
-
- _visitPattern(pattern, name) {
- if (!pattern.startsWith('^') || !pattern.endsWith('$')) {
- throw new Error('Pattern must start with "^" and end with "$"');
- }
- pattern = pattern.slice(1, -1);
- const subRuleIds = {};
-
- let i = 0;
- const length = pattern.length;
-
- const getDot = () => {
- let rule;
- if (this._dotall) {
- rule = '[\\U00000000-\\U0010FFFF]';
- } else {
- // Accept any character... except \n and \r line break chars (\x0A and \xOD)
- rule = '[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]';
- }
- return this._addRule('dot', rule);
- };
-
-
- const toRule = ([s, isLiteral]) => isLiteral ? "\"" + s + "\"" : s;
-
- const transform = () => {
- const start = i;
- // For each component of this sequence, store its string representation and whether it's a literal.
- // We only need a flat structure here to apply repetition operators to the last item, and
- // to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially
- // (GBNF's syntax is luckily very close to regular expressions!)
- const seq = [];
-
- const joinSeq = () => {
- const ret = [];
- for (const [isLiteral, g] of groupBy(seq, x => x[1])) {
- if (isLiteral) {
- ret.push([[...g].map(x => x[0]).join(''), true]);
- } else {
- ret.push(...g);
- }
- }
- if (ret.length === 1) {
- return ret[0];
- }
- return [ret.map(x => toRule(x)).join(' '), false];
- };
-
- while (i < length) {
- const c = pattern[i];
- if (c === '.') {
- seq.push([getDot(), false]);
- i += 1;
- } else if (c === '(') {
- i += 1;
- if (i < length) {
- if (pattern[i] === '?') {
- throw new Error(`Unsupported pattern syntax "${pattern[i]}" at index ${i} of /${pattern}/`);
- }
- }
- seq.push([`(${toRule(transform())})`, false]);
- } else if (c === ')') {
- i += 1;
- if (start <= 0 || pattern[start - 1] !== '(') {
- throw new Error(`Unbalanced parentheses; start = ${start}, i = ${i}, pattern = ${pattern}`);
- }
- return joinSeq();
- } else if (c === '[') {
- let squareBrackets = c;
- i += 1;
- while (i < length && pattern[i] !== ']') {
- if (pattern[i] === '\\') {
- squareBrackets += pattern.slice(i, i + 2);
- i += 2;
- } else {
- squareBrackets += pattern[i];
- i += 1;
- }
- }
- if (i >= length) {
- throw new Error(`Unbalanced square brackets; start = ${start}, i = ${i}, pattern = ${pattern}`);
- }
- squareBrackets += ']';
- i += 1;
- seq.push([squareBrackets, false]);
- } else if (c === '|') {
- seq.push(['|', false]);
- i += 1;
- } else if (c === '*' || c === '+' || c === '?') {
- seq[seq.length - 1] = [toRule(seq[seq.length - 1]) + c, false];
- i += 1;
- } else if (c === '{') {
- let curlyBrackets = c;
- i += 1;
- while (i < length && pattern[i] !== '}') {
- curlyBrackets += pattern[i];
- i += 1;
- }
- if (i >= length) {
- throw new Error(`Unbalanced curly brackets; start = ${start}, i = ${i}, pattern = ${pattern}`);
- }
- curlyBrackets += '}';
- i += 1;
- const nums = curlyBrackets.slice(1, -1).split(',').map(s => s.trim());
- let minTimes, maxTimes;
- if (nums.length === 1) {
- minTimes = parseInt(nums[0], 10);
- maxTimes = minTimes;
- } else {
- if (nums.length !== 2) {
- throw new Error(`Invalid quantifier ${curlyBrackets}`);
- }
- minTimes = nums[0] ? parseInt(nums[0], 10) : 0;
- maxTimes = nums[1] ? parseInt(nums[1], 10) : Infinity;
- }
-
- let [sub, subIsLiteral] = seq[seq.length - 1];
-
- if (minTimes === 0 && maxTimes === Infinity) {
- seq[seq.length - 1] = [`${sub}*`, false];
- } else if (minTimes === 0 && maxTimes === 1) {
- seq[seq.length - 1] = [`${sub}?`, false];
- } else if (minTimes === 1 && maxTimes === Infinity) {
- seq[seq.length - 1] = [`${sub}+`, false];
- } else {
- if (!subIsLiteral) {
- let id = subRuleIds[sub];
- if (id === undefined) {
- id = this._addRule(`${name}-${Object.keys(subRuleIds).length + 1}`, sub);
- subRuleIds[sub] = id;
- }
- sub = id;
- }
-
- const repeatedSub = Array.from({ length: minTimes }, () => subIsLiteral ? `"${sub.slice(1, -1).repeat(minTimes)}"` : sub);
- const optionalSub = maxTimes !== undefined ? Array.from({ length: maxTimes - minTimes }, () => `${sub}?`) : [`${sub}*`];
- seq[seq.length - 1] = [repeatedSub.concat(optionalSub).join(' '), false];
- }
- } else {
- let literal = '';
- while (i < length) {
- if (pattern[i] === '\\' && i < length - 1) {
- const next = pattern[i + 1];
- if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.has(next)) {
- i += 1;
- literal += pattern[i];
- i += 1;
- } else {
- literal += pattern.slice(i, i + 2);
- i += 2;
- }
- } else if (pattern[i] === '"') {
- literal += '\\"';
- i += 1;
- } else if (!NON_LITERAL_SET.has(pattern[i]) &&
- (i === length - 1 || literal === '' || pattern[i + 1] === '.' || !NON_LITERAL_SET.has(pattern[i+1]))) {
- literal += pattern[i];
- i += 1;
- } else {
- break;
- }
- }
- if (literal !== '') {
- seq.push([literal, true]);
- }
- }
- }
-
- return joinSeq();
- };
-
- return this._addRule(name, "\"\\\"\" " + toRule(transform()) + " \"\\\"\" space")
- }
-
- _resolveRef(ref) {
- let refName = ref.split('/').pop();
- if (!(refName in this._rules) && !this._refsBeingResolved.has(ref)) {
- this._refsBeingResolved.add(ref);
- const resolved = this._refs[ref];
- refName = this.visit(resolved, refName);
- this._refsBeingResolved.delete(ref);
- }
- return refName;
- }
-
- _generateConstantRule(value) {
- if (typeof value !== 'string') {
- throw new Error('Only string constants are supported, got ' + JSON.stringify(value));
- }
- return this._formatLiteral(value);
- }
-
visit(schema, name) {
const schemaType = schema.type;
- const schemaFormat = schema.format;
- const ruleName = name in RESERVED_NAMES ? name + '-' : name == '' ? 'root' : name;
+ const ruleName = name || 'root';
+
+ if (schema.oneOf || schema.anyOf) {
+ const rule = (schema.oneOf || schema.anyOf).map((altSchema, i) =>
+ this.visit(altSchema, `${name}${name ? "-" : ""}${i}`)
+ ).join(' | ');
- const ref = schema.$ref;
- if (ref !== undefined) {
- return this._addRule(ruleName, this._resolveRef(ref));
- } else if (schema.oneOf || schema.anyOf) {
- return this._addRule(ruleName, this._generateUnionRule(name, schema.oneOf || schema.anyOf));
- } else if (Array.isArray(schemaType)) {
- return this._addRule(ruleName, this._generateUnionRule(name, schemaType.map(t => ({ type: t }))));
+ return this._addRule(ruleName, rule);
} else if ('const' in schema) {
- if (typeof schema.const !== 'string') {
- throw new Error('Only string constants are supported, got ' + JSON.stringify(schema.const));
- }
- return this._addRule(ruleName, this._generateConstantRule(schema.const));
+ return this._addRule(ruleName, this._formatLiteral(schema.const));
} else if ('enum' in schema) {
- const rule = schema.enum.map(v => this._generateConstantRule(v)).join(' | ');
+ const rule = schema.enum.map(v => this._formatLiteral(v)).join(' | ');
return this._addRule(ruleName, rule);
- } else if ((schemaType === undefined || schemaType === 'object') &&
- ('properties' in schema ||
- ('additionalProperties' in schema && schema.additionalProperties !== true))) {
- const required = new Set(schema.required || []);
- const properties = Object.entries(schema.properties ?? {});
- return this._addRule(ruleName, this._buildObjectRule(properties, required, name, schema.additionalProperties));
- } else if ((schemaType === undefined || schemaType === 'object') && 'allOf' in schema) {
- const required = new Set();
- const properties = [];
- const addComponent = (compSchema, isRequired) => {
- const ref = compSchema.$ref;
- if (ref !== undefined) {
- compSchema = this._refs[ref];
+ } else if (schemaType === 'object' && 'properties' in schema) {
+ // TODO: `required` keyword (from python implementation)
+ const propOrder = this._propOrder;
+ const propPairs = Object.entries(schema.properties).sort((a, b) => {
+ // sort by position in prop_order (if specified) then by key
+ const orderA = typeof propOrder[a[0]] === 'number' ? propOrder[a[0]] : Infinity;
+ const orderB = typeof propOrder[b[0]] === 'number' ? propOrder[b[0]] : Infinity;
+ return orderA - orderB || a[0].localeCompare(b[0]);
+ });
+
+ let rule = '"{" space';
+ propPairs.forEach(([propName, propSchema], i) => {
+ const propRuleName = this.visit(propSchema, `${name}${name ? "-" : ""}${propName}`);
+ if (i > 0) {
+ rule += ' "," space';
}
+ rule += ` ${this._formatLiteral(propName)} space ":" space ${propRuleName}`;
+ });
+ rule += ' "}" space';
- if ('properties' in compSchema) {
- for (const [propName, propSchema] of Object.entries(compSchema.properties)) {
- properties.push([propName, propSchema]);
- if (isRequired) {
- required.add(propName);
- }
- }
- }
- };
-
- for (const t of schema.allOf) {
- if ('anyOf' in t) {
- for (const tt of t.anyOf) {
- addComponent(tt, false);
- }
- } else {
- addComponent(t, true);
- }
- }
-
- return this._addRule(ruleName, this._buildObjectRule(properties, required, name, /* additionalProperties= */ false));
- } else if ((schemaType === undefined || schemaType === 'array') && ('items' in schema || 'prefixItems' in schema)) {
- const items = schema.items ?? schema.prefixItems;
- if (Array.isArray(items)) {
- return this._addRule(
- ruleName,
- '"[" space ' +
- items.map((item, i) => this.visit(item, `${name ?? ''}${name ? '-' : ''}tuple-${i}`)).join(' "," space ') +
- ' "]" space'
- );
- } else {
- const itemRuleName = this.visit(items, `${name ?? ''}${name ? '-' : ''}item`);
- const listItemOperator = `( "," space ${itemRuleName} )`;
- let successiveItems = '';
- let minItems = schema.minItems || 0;
- const maxItems = schema.maxItems;
- if (minItems > 0) {
- successiveItems = listItemOperator.repeat(minItems - 1);
- minItems--;
- }
- if (maxItems !== undefined && maxItems > minItems) {
- successiveItems += `${listItemOperator}?`.repeat(maxItems - minItems - 1);
- } else {
- successiveItems += `${listItemOperator}*`;
- }
- const rule = minItems === 0
- ? `"[" space ( ${itemRuleName} ${successiveItems} )? "]" space`
- : `"[" space ${itemRuleName} ${successiveItems} "]" space`;
- return this._addRule(ruleName, rule);
- }
- } else if ((schemaType === undefined || schemaType === 'string') && 'pattern' in schema) {
- return this._visitPattern(schema.pattern, ruleName);
- } else if ((schemaType === undefined || schemaType === 'string') && /^uuid[1-5]?$/.test(schema.format || '')) {
- return this._addRule(
- ruleName === 'root' ? 'root' : schemaFormat,
- PRIMITIVE_RULES['uuid'])
- } else if ((schemaType === undefined || schemaType === 'string') && schema.format in DATE_RULES) {
- for (const [t, r] of Object.entries(DATE_RULES)) {
- this._addRule(t, r);
- }
- return schemaFormat + '-string';
- } else if ((schemaType === 'object') || (Object.keys(schema).length === 0)) {
- for (const n of OBJECT_RULE_NAMES) {
- this._addRule(n, PRIMITIVE_RULES[n]);
- }
- return this._addRule(ruleName, 'object');
+ return this._addRule(ruleName, rule);
+ } else if (schemaType === 'array' && 'items' in schema) {
+ // TODO `prefixItems` keyword (from python implementation)
+ const itemRuleName = this.visit(schema.items, `${name}${name ? "-" : ""}item`);
+ const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`;
+ return this._addRule(ruleName, rule);
} else {
- if (!(schemaType in PRIMITIVE_RULES)) {
+ if (!PRIMITIVE_RULES[schemaType]) {
throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`);
}
- // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
- return this._addRule(ruleName === 'root' ? 'root' : schemaType, PRIMITIVE_RULES[schemaType]);
- }
- }
-
- _buildObjectRule(properties, required, name, additionalProperties) {
- const propOrder = this._propOrder;
- // sort by position in prop_order (if specified) then by original order
- const sortedProps = properties.map(([k]) => k).sort((a, b) => {
- const orderA = propOrder[a] || Infinity;
- const orderB = propOrder[b] || Infinity;
- return orderA - orderB || properties.findIndex(([k]) => k === a) - properties.findIndex(([k]) => k === b);
- });
-
- const propKvRuleNames = {};
- for (const [propName, propSchema] of properties) {
- const propRuleName = this.visit(propSchema, `${name ?? ''}${name ? '-' : ''}${propName}`);
- propKvRuleNames[propName] = this._addRule(
- `${name ?? ''}${name ? '-' : ''}${propName}-kv`,
- `${this._formatLiteral(propName)} space ":" space ${propRuleName}`
+ return this._addRule(
+ ruleName === 'root' ? 'root' : schemaType,
+ PRIMITIVE_RULES[schemaType]
);
}
- const requiredProps = sortedProps.filter(k => required.has(k));
- const optionalProps = sortedProps.filter(k => !required.has(k));
-
- if (typeof additionalProperties === 'object' || additionalProperties === true) {
- const subName = `${name ?? ''}${name ? '-' : ''}additional`;
- const valueRule = this.visit(additionalProperties === true ? {} : additionalProperties, `${subName}-value`);
- propKvRuleNames['*'] = this._addRule(
- `${subName}-kv`,
- `${this._addRule('string', PRIMITIVE_RULES['string'])} ":" space ${valueRule}`);
- optionalProps.push('*');
- }
-
- let rule = '"{" space ';
- rule += requiredProps.map(k => propKvRuleNames[k]).join(' "," space ');
-
- if (optionalProps.length > 0) {
- rule += ' (';
- if (requiredProps.length > 0) {
- rule += ' "," space ( ';
- }
-
- const getRecursiveRefs = (ks, firstIsOptional) => {
- const [k, ...rest] = ks;
- const kvRuleName = propKvRuleNames[k];
- let res;
- if (k === '*') {
- res = this._addRule(
- `${name ?? ''}${name ? '-' : ''}additional-kvs`,
- `${kvRuleName} ( "," space ` + kvRuleName + ` )*`
- )
- } else if (firstIsOptional) {
- res = `( "," space ${kvRuleName} )?`;
- } else {
- res = kvRuleName;
- }
- if (rest.length > 0) {
- res += ' ' + this._addRule(
- `${name ?? ''}${name ? '-' : ''}${k}-rest`,
- getRecursiveRefs(rest, true)
- );
- }
- return res;
- };
-
- rule += optionalProps.map((_, i) => getRecursiveRefs(optionalProps.slice(i), false)).join(' | ');
- if (requiredProps.length > 0) {
- rule += ' )';
- }
- rule += ' )?';
- }
-
- rule += ' "}" space';
-
- return rule;
}
formatGrammar() {
let grammar = '';
- for (const [name, rule] of Object.entries(this._rules).sort(([a], [b]) => a.localeCompare(b))) {
+ this._rules.forEach((rule, name) => {
grammar += `${name} ::= ${rule}\n`;
- }
+ });
return grammar;
}
}
-
-// Helper function to group elements by a key function
-function* groupBy(iterable, keyFn) {
- let lastKey = null;
- let group = [];
- for (const element of iterable) {
- const key = keyFn(element);
- if (lastKey !== null && key !== lastKey) {
- yield [lastKey, group];
- group = [];
- }
- group.push(element);
- lastKey = key;
- }
- if (group.length > 0) {
- yield [lastKey, group];
- }
-}
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index 914482ab..4d3a3e27 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -1,141 +1,152 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#include "utils.h"
-#include "macsandbox.h"
-
#include "llama.cpp/common.h"
#include "llama.cpp/llama.h"
#include "llama.cpp/grammar-parser.h"
-#include "llama.cpp/json-schema-to-grammar.h"
+#include "llama.cpp/llava/llava.h"
+#include "llama.cpp/stb_image.h"
+#include "utils.h"
+#include "oai.h"
+#include "macsandbox.h"
// increase max payload length to allow use of larger context size
#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576
#include "httplib.h"
#include "llama.cpp/json.h"
-#include <atomic>
-#include <chrono>
-#include <condition_variable>
#include <cstddef>
-#include <set>
-#include <mutex>
#include <thread>
+#include <chrono>
+#include <condition_variable>
+#include <atomic>
#include <signal.h>
-#include <memory>
#include <libc/calls/pledge.h>
#include <tool/args/args.h>
#include <libc/dce.h>
using json = nlohmann::json;
-bool server_verbose = false;
-bool server_log_json = true;
-
-enum stop_type {
- STOP_TYPE_FULL,
- STOP_TYPE_PARTIAL,
-};
-
-enum slot_state {
- SLOT_STATE_IDLE,
- SLOT_STATE_PROCESSING,
-};
-
-enum slot_command {
- SLOT_COMMAND_NONE,
- SLOT_COMMAND_LOAD_PROMPT,
- SLOT_COMMAND_RELEASE,
-};
-
-enum server_state {
- SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet
- SERVER_STATE_READY, // Server is ready and model is loaded
- SERVER_STATE_ERROR // An error occurred, load_model failed
-};
-
-enum server_task_type {
- SERVER_TASK_TYPE_COMPLETION,
- SERVER_TASK_TYPE_CANCEL,
- SERVER_TASK_TYPE_NEXT_RESPONSE,
- SERVER_TASK_TYPE_METRICS
-};
-
-struct server_task {
- int id = -1; // to be filled by server_queue
- int id_multi = -1;
- int id_target = -1;
-
- server_task_type type;
- json data;
-
- bool infill = false;
- bool embedding = false;
-};
-
-struct server_task_result {
- int id = -1;
- int id_multi = -1;
-
- json data;
-
- bool stop;
- bool error;
-};
-
-struct server_task_multi {
- int id = -1;
-
- std::set<int> subtasks_remaining;
- std::vector<server_task_result> results;
+struct server_params
+{
+ std::string hostname = "127.0.0.1";
+ std::vector<std::string> api_keys;
+ std::string public_path = "/zip/llama.cpp/server/public";
+ std::string chat_template = "";
+ int32_t port = 8080;
+ int32_t read_timeout = 600;
+ int32_t write_timeout = 600;
+ bool nobrowser = false;
+ bool slots_endpoint = true;
+ bool metrics_endpoint = false;
};
-struct slot_params {
- bool stream = true;
- bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt
-
- uint32_t seed = -1; // RNG seed
- int32_t n_keep = 0; // number of tokens to keep from initial prompt
- int32_t n_predict = -1; // new tokens to predict
+bool server_verbose = false;
+bool server_log_json = true;
- std::vector<std::string> antiprompt;
+static size_t common_part(const std::vector<llama_token> &a, const std::vector<llama_token> &b)
+{
+ size_t i;
+ for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++)
+ {
+ }
+ return i;
+}
- json input_prefix;
- json input_suffix;
+enum stop_type
+{
+ STOP_FULL,
+ STOP_PARTIAL,
};
-struct server_params {
- int32_t port = 8080;
- int32_t read_timeout = 600;
- int32_t write_timeout = 600;
- int32_t n_threads_http = -1;
-
- std::string hostname = "127.0.0.1";
- std::string public_path = "/zip/llama.cpp/server/public";
- std::string chat_template = "";
- std::string system_prompt = "";
+static bool ends_with(const std::string &str, const std::string &suffix)
+{
+ return str.size() >= suffix.size() &&
+ 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
+}
- std::vector<std::string> api_keys;
+static size_t find_partial_stop_string(const std::string &stop,
+ const std::string &text)
+{
+ if (!text.empty() && !stop.empty())
+ {
+ const char text_last_char = text.back();
+ for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--)
+ {
+ if (stop[char_index] == text_last_char)
+ {
+ const std::string current_partial = stop.substr(0, char_index + 1);
+ if (ends_with(text, current_partial))
+ {
+ return text.size() - char_index - 1;
+ }
+ }
+ }
+ }
+ return std::string::npos;
+}
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- std::string ssl_key_file = "";
- std::string ssl_cert_file = "";
-#endif
+// TODO: reuse llama_detokenize
+template <class Iter>
+static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end)
+{
+ std::string ret;
+ for (; begin != end; ++begin)
+ {
+ ret += llama_token_to_piece(ctx, *begin);
+ }
+ return ret;
+}
- bool slots_endpoint = true;
- bool metrics_endpoint = false;
+// format incomplete utf-8 multibyte character for output
+static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token)
+{
+ std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token);
+ // if the size is 1 and first bit is 1, meaning it's a partial character
+ // (size > 1 meaning it's already a known token)
+ if (out.size() == 1 && (out[0] & 0x80) == 0x80)
+ {
+ std::stringstream ss;
+ ss << std::hex << (out[0] & 0xff);
+ std::string res(ss.str());
+ out = "byte: \\x" + res;
+ }
+ return out;
+}
- bool nobrowser = false;
-};
+// convert a vector of completion_token_output to json
+static json probs_vector_to_json(const llama_context *ctx, const std::vector<completion_token_output> &probs)
+{
+ json out = json::array();
+ for (const auto &prob : probs)
+ {
+ json probs_for_token = json::array();
+ for (const auto &p : prob.probs)
+ {
+ std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);
+ probs_for_token.push_back(json
+ {
+ {"tok_str", tok_str},
+ {"prob", p.prob},
+ });
+ }
+ std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);
+ out.push_back(json{
+ {"content", tok_str},
+ {"probs", probs_for_token},
+ });
+ }
+ return out;
+}
-struct server_slot {
+struct llama_client_slot
+{
int id;
- int id_task = -1;
- int id_multi = -1;
+ int task_id = -1;
struct slot_params params;
- slot_state state = SLOT_STATE_IDLE;
- slot_command command = SLOT_COMMAND_NONE;
+ slot_state state = IDLE;
+ slot_command command = NONE;
// used to determine the slot that has been used the longest
int64_t t_last_used = -1;
@@ -146,38 +157,33 @@ struct server_slot {
int32_t n_decoded = 0;
int32_t n_remaining = -1;
int32_t i_batch = -1;
- int32_t n_predict = -1; // TODO: disambiguate from params.n_predict
+ int32_t n_predict = -1;
- int32_t n_prompt_tokens = 0;
- int32_t n_prompt_tokens_processed = 0;
+ int32_t num_prompt_tokens = 0;
+ int32_t num_prompt_tokens_processed = 0;
json prompt;
-
- // when a task is submitted, we first tokenize the prompt and store it here
- std::vector<llama_token> prompt_tokens;
-
std::string generated_text;
+ llama_token sampled;
std::vector<llama_token> cache_tokens;
std::vector<completion_token_output> generated_token_probs;
- bool infill = false;
- bool embedding = false;
+ bool infill = false;
+ bool embedding = false;
bool has_next_token = true;
- bool truncated = false;
- bool stopped_eos = false;
- bool stopped_word = false;
- bool stopped_limit = false;
+ bool truncated = false;
+ bool stopped_eos = false;
+ bool stopped_word = false;
+ bool stopped_limit = false;
bool oaicompat = false;
-
std::string oaicompat_model;
+
std::string stopping_word;
// sampling
- llama_token sampled;
struct llama_sampling_params sparams;
- llama_sampling_context * ctx_sampling = nullptr;
- json json_schema;
+ llama_sampling_context *ctx_sampling = nullptr;
int32_t ga_i = 0; // group-attention state
int32_t ga_n = 1; // group-attention factor
@@ -185,44 +191,65 @@ struct server_slot {
int32_t n_past_se = 0; // self-extend
+ // multimodal
+ std::vector<slot_image> images;
+
// stats
- size_t n_sent_text = 0; // number of sent text character
- size_t n_sent_token_probs = 0;
+ size_t sent_count = 0;
+ size_t sent_token_probs_index = 0;
int64_t t_start_process_prompt;
- int64_t t_start_generation;
+ int64_t t_start_genereration;
double t_prompt_processing; // ms
double t_token_generation; // ms
+ // multitasks
+ int multitask_id = -1;
+
void reset() {
- n_prompt_tokens = 0;
- generated_text = "";
- truncated = false;
- stopped_eos = false;
- stopped_word = false;
- stopped_limit = false;
- stopping_word = "";
- n_past = 0;
- n_sent_text = 0;
- n_sent_token_probs = 0;
- infill = false;
- ga_i = 0;
- n_past_se = 0;
+ num_prompt_tokens = 0;
+ generated_text = "";
+ truncated = false;
+ stopped_eos = false;
+ stopped_word = false;
+ stopped_limit = false;
+ stopping_word = "";
+ n_past = 0;
+ sent_count = 0;
+ sent_token_probs_index = 0;
+ infill = false;
+ ga_i = 0;
+ n_past_se = 0;
generated_token_probs.clear();
+
+ for (slot_image & img : images)
+ {
+ free(img.image_embedding);
+ if (img.img_data) {
+ clip_image_u8_free(img.img_data);
+ }
+ img.prefix_prompt = "";
+ }
+
+ images.clear();
}
bool has_budget(gpt_params &global_params) {
- if (params.n_predict == -1 && global_params.n_predict == -1) {
+ if (params.n_predict == -1 && global_params.n_predict == -1)
+ {
return true; // limitless
}
n_remaining = -1;
- if (params.n_predict != -1) {
+ if (params.n_predict != -1)
+ {
n_remaining = params.n_predict - n_decoded;
- } else if (global_params.n_predict != -1) {
+ }
+ else if (global_params.n_predict != -1)
+ {
n_remaining = global_params.n_predict - n_decoded;
}
@@ -230,33 +257,37 @@ struct server_slot {
}
bool available() const {
- return state == SLOT_STATE_IDLE && command == SLOT_COMMAND_NONE;
+ return state == IDLE && command == NONE;
}
bool is_processing() const {
- return (state == SLOT_STATE_IDLE && command == SLOT_COMMAND_LOAD_PROMPT) || state == SLOT_STATE_PROCESSING;
+ return (state == IDLE && command == LOAD_PROMPT) || state == PROCESSING;
}
- void add_token_string(const completion_token_output & token) {
- if (command == SLOT_COMMAND_RELEASE) {
+ void add_token_string(const completion_token_output &token) {
+ if (command == RELEASE)
+ {
return;
}
+ cache_tokens.push_back(token.tok);
generated_token_probs.push_back(token);
}
void release() {
- if (state == SLOT_STATE_PROCESSING) {
- t_token_generation = (ggml_time_us() - t_start_generation) / 1e3;
- command = SLOT_COMMAND_RELEASE;
+ if (state == PROCESSING)
+ {
+ t_token_generation = (ggml_time_us() - t_start_genereration) / 1e3;
+ command = RELEASE;
}
}
- json get_formated_timings() const {
- return json {
- {"prompt_n", n_prompt_tokens_processed},
+ json get_formated_timings() {
+ return json
+ {
+ {"prompt_n", num_prompt_tokens_processed},
{"prompt_ms", t_prompt_processing},
- {"prompt_per_token_ms", t_prompt_processing / n_prompt_tokens_processed},
- {"prompt_per_second", 1e3 / t_prompt_processing * n_prompt_tokens_processed},
+ {"prompt_per_token_ms", t_prompt_processing / num_prompt_tokens_processed},
+ {"prompt_per_second", 1e3 / t_prompt_processing * num_prompt_tokens_processed},
{"predicted_n", n_decoded},
{"predicted_ms", t_token_generation},
@@ -265,74 +296,40 @@ struct server_slot {
};
}
- size_t find_stopping_strings(const std::string & text, const size_t last_token_size, const stop_type type) {
- size_t stop_pos = std::string::npos;
-
- for (const std::string & word : params.antiprompt) {
- size_t pos;
-
- if (type == STOP_TYPE_FULL) {
- const size_t tmp = word.size() + last_token_size;
- const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;
-
- pos = text.find(word, from_pos);
- } else {
- pos = find_partial_stop_string(word, text);
- }
-
- if (pos != std::string::npos && (stop_pos == std::string::npos || pos < stop_pos)) {
- if (type == STOP_TYPE_FULL) {
- stopped_word = true;
- stopping_word = word;
- has_next_token = false;
- }
- stop_pos = pos;
- }
- }
-
- return stop_pos;
- }
-
void print_timings() const {
- char buffer[512];
-
- double t_token = t_prompt_processing / n_prompt_tokens_processed;
- double n_tokens_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;
-
- snprintf(buffer, 512, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)",
- t_prompt_processing, n_prompt_tokens_processed,
+ char buffer[512];
+ double t_token = t_prompt_processing / num_prompt_tokens_processed;
+ double n_tokens_second = 1e3 / t_prompt_processing * num_prompt_tokens_processed;
+ sprintf(buffer, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)",
+ t_prompt_processing, num_prompt_tokens_processed,
t_token, n_tokens_second);
-
LOG_INFO(buffer, {
- {"id_slot", id},
- {"id_task", id_task},
- {"t_prompt_processing", t_prompt_processing},
- {"n_prompt_tokens_processed", n_prompt_tokens_processed},
- {"t_token", t_token},
- {"n_tokens_second", n_tokens_second},
+ {"slot_id", id},
+ {"task_id", task_id},
+ {"t_prompt_processing", t_prompt_processing},
+ {"num_prompt_tokens_processed", num_prompt_tokens_processed},
+ {"t_token", t_token},
+ {"n_tokens_second", n_tokens_second},
});
t_token = t_token_generation / n_decoded;
n_tokens_second = 1e3 / t_token_generation * n_decoded;
-
- snprintf(buffer, 512, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)",
+ sprintf(buffer, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)",
t_token_generation, n_decoded,
t_token, n_tokens_second);
-
LOG_INFO(buffer, {
- {"id_slot", id},
- {"id_task", id_task},
+ {"slot_id", id},
+ {"task_id", task_id},
{"t_token_generation", t_token_generation},
{"n_decoded", n_decoded},
{"t_token", t_token},
{"n_tokens_second", n_tokens_second},
});
- snprintf(buffer, 512, " total time = %10.2f ms", t_prompt_processing + t_token_generation);
-
+ sprintf(buffer, " total time = %10.2f ms", t_prompt_processing + t_token_generation);
LOG_INFO(buffer, {
- {"id_slot", id},
- {"id_task", id_task},
+ {"slot_id", id},
+ {"task_id", task_id},
{"t_prompt_processing", t_prompt_processing},
{"t_token_generation", t_token_generation},
{"t_total", t_prompt_processing + t_token_generation},
@@ -340,36 +337,29 @@ struct server_slot {
}
};
-struct server_metrics {
- int64_t t_start = 0;
-
+struct llama_metrics {
uint64_t n_prompt_tokens_processed_total = 0;
- uint64_t t_prompt_processing_total = 0;
uint64_t n_tokens_predicted_total = 0;
- uint64_t t_tokens_generation_total = 0;
uint64_t n_prompt_tokens_processed = 0;
uint64_t t_prompt_processing = 0;
- uint64_t n_tokens_predicted = 0;
- uint64_t t_tokens_generation = 0;
+ uint64_t n_tokens_predicted = 0;
+ uint64_t t_tokens_generation = 0;
- void init() {
- t_start = ggml_time_us();
- }
- void on_prompt_eval(const server_slot & slot) {
- n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed;
- n_prompt_tokens_processed += slot.n_prompt_tokens_processed;
- t_prompt_processing += slot.t_prompt_processing;
- t_prompt_processing_total += slot.t_prompt_processing;
+ void on_prompt_eval(const llama_client_slot &slot) {
+ n_prompt_tokens_processed_total += slot.num_prompt_tokens_processed;
+
+ n_prompt_tokens_processed += slot.num_prompt_tokens_processed;
+ t_prompt_processing += slot.t_prompt_processing;
}
- void on_prediction(const server_slot & slot) {
- n_tokens_predicted_total += slot.n_decoded;
- n_tokens_predicted += slot.n_decoded;
- t_tokens_generation += slot.t_token_generation;
- t_tokens_generation_total += slot.t_token_generation;
+ void on_prediction(const llama_client_slot &slot) {
+ n_tokens_predicted_total += slot.n_decoded;
+
+ n_tokens_predicted += slot.n_decoded;
+ t_tokens_generation += slot.t_token_generation;
}
void reset_bucket() {
@@ -380,261 +370,23 @@ struct server_metrics {
}
};
-struct server_queue {
- int id = 0;
- bool running;
-
- // queues
- std::vector<server_task> queue_tasks;
- std::vector<server_task> queue_tasks_deferred;
-
- std::vector<server_task_multi> queue_multitasks;
-
- std::mutex mutex_tasks;
- std::condition_variable condition_tasks;
-
- // callback functions
- std::function<void(server_task &)> callback_new_task;
- std::function<void(server_task_multi &)> callback_finish_multitask;
- std::function<void(void)> callback_update_slots;
-
- // Add a new task to the end of the queue
- int post(server_task task) {
- std::unique_lock<std::mutex> lock(mutex_tasks);
- if (task.id == -1) {
- task.id = id++;
- LOG_VERBOSE("new task id", {{"new_id", task.id}});
- }
- queue_tasks.push_back(std::move(task));
- condition_tasks.notify_one();
- return task.id;
- }
-
- // Add a new task, but defer until one slot is available
- void defer(server_task task) {
- std::unique_lock<std::mutex> lock(mutex_tasks);
- queue_tasks_deferred.push_back(std::move(task));
- }
-
- // Get the next id for creating anew task
- int get_new_id() {
- std::unique_lock<std::mutex> lock(mutex_tasks);
- int new_id = id++;
- LOG_VERBOSE("new task id", {{"new_id", new_id}});
- return new_id;
- }
-
- // Register function to process a new task
- void on_new_task(std::function<void(server_task &)> callback) {
- callback_new_task = std::move(callback);
- }
-
- // Register function to process a multitask when it is finished
- void on_finish_multitask(std::function<void(server_task_multi&)> callback) {
- callback_finish_multitask = std::move(callback);
- }
-
- // Register the function to be called when all slots data is ready to be processed
- void on_update_slots(std::function<void(void)> callback) {
- callback_update_slots = std::move(callback);
- }
-
- // Call when the state of one slot is changed
- void notify_slot_changed() {
- // move deferred tasks back to main loop
- std::unique_lock<std::mutex> lock(mutex_tasks);
- for (auto & task : queue_tasks_deferred) {
- queue_tasks.push_back(std::move(task));
- }
- queue_tasks_deferred.clear();
- }
-
- // end the start_loop routine
- void terminate() {
- std::unique_lock<std::mutex> lock(mutex_tasks);
- running = false;
- condition_tasks.notify_all();
- }
-
- /**
- * Main loop consists of these steps:
- * - Wait until a new task arrives
- * - Process the task (i.e. maybe copy data into slot)
- * - Check if multitask is finished
- * - Update all slots
- */
- void start_loop() {
- running = true;
-
- while (true) {
- LOG_VERBOSE("new task may arrive", {});
-
- while (true) {
- std::unique_lock<std::mutex> lock(mutex_tasks);
- if (queue_tasks.empty()) {
- lock.unlock();
- break;
- }
- server_task task = queue_tasks.front();
- queue_tasks.erase(queue_tasks.begin());
- lock.unlock();
- LOG_VERBOSE("callback_new_task", {{"id_task", task.id}});
- callback_new_task(task);
- }
-
- LOG_VERBOSE("update_multitasks", {});
-
- // check if we have any finished multitasks
- auto queue_iterator = queue_multitasks.begin();
- while (queue_iterator != queue_multitasks.end()) {
- if (queue_iterator->subtasks_remaining.empty()) {
- // all subtasks done == multitask is done
- server_task_multi current_multitask = *queue_iterator;
- callback_finish_multitask(current_multitask);
- // remove this multitask
- queue_iterator = queue_multitasks.erase(queue_iterator);
- } else {
- ++queue_iterator;
- }
- }
-
- // all tasks in the current loop is processed, slots data is now ready
- LOG_VERBOSE("callback_update_slots", {});
-
- callback_update_slots();
-
- LOG_VERBOSE("wait for new task", {});
- {
- std::unique_lock<std::mutex> lock(mutex_tasks);
- if (queue_tasks.empty()) {
- if (!running) {
- LOG_VERBOSE("ending start_loop", {});
- return;
- }
- condition_tasks.wait(lock, [&]{
- return (!queue_tasks.empty() || !running);
- });
- }
- }
- }
- }
-
- //
- // functions to manage multitasks
- //
-
- // add a multitask by specifying the id of all subtask (subtask is a server_task)
- void add_multitask(int id_multi, std::vector<int> & sub_ids) {
- std::lock_guard<std::mutex> lock(mutex_tasks);
- server_task_multi multi;
- multi.id = id_multi;
- std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));
- queue_multitasks.push_back(multi);
- }
-
- // updatethe remaining subtasks, while appending results to multitask
- void update_multitask(int id_multi, int id_sub, server_task_result & result) {
- std::lock_guard<std::mutex> lock(mutex_tasks);
- for (auto & multitask : queue_multitasks) {
- if (multitask.id == id_multi) {
- multitask.subtasks_remaining.erase(id_sub);
- multitask.results.push_back(result);
- }
- }
- }
-};
-
-struct server_response {
- typedef std::function<void(int, int, server_task_result &)> callback_multitask_t;
- callback_multitask_t callback_update_multitask;
-
- // for keeping track of all tasks waiting for the result
- std::set<int> waiting_task_ids;
-
- // the main result queue
- std::vector<server_task_result> queue_results;
-
- std::mutex mutex_results;
- std::condition_variable condition_results;
-
- // add the id_task to the list of tasks waiting for response
- void add_waiting_task_id(int id_task) {
- LOG_VERBOSE("waiting for task id", {{"id_task", id_task}});
-
- std::unique_lock<std::mutex> lock(mutex_results);
- waiting_task_ids.insert(id_task);
- }
-
- // when the request is finished, we can remove task associated with it
- void remove_waiting_task_id(int id_task) {
- LOG_VERBOSE("remove waiting for task id", {{"id_task", id_task}});
-
- std::unique_lock<std::mutex> lock(mutex_results);
- waiting_task_ids.erase(id_task);
- }
-
- // This function blocks the thread until there is a response for this id_task
- server_task_result recv(int id_task) {
- while (true) {
- std::unique_lock<std::mutex> lock(mutex_results);
- condition_results.wait(lock, [&]{
- return !queue_results.empty();
- });
-
- for (int i = 0; i < (int) queue_results.size(); i++) {
- if (queue_results[i].id == id_task) {
- assert(queue_results[i].id_multi == -1);
- server_task_result res = queue_results[i];
- queue_results.erase(queue_results.begin() + i);
- return res;
- }
- }
- }
-
- // should never reach here
- }
-
- // Register the function to update multitask
- void on_multitask_update(callback_multitask_t callback) {
- callback_update_multitask = std::move(callback);
- }
-
- // Send a new result to a waiting id_task
- void send(server_task_result result) {
- LOG_VERBOSE("send new result", {{"id_task", result.id}});
-
- std::unique_lock<std::mutex> lock(mutex_results);
- for (const auto & id_task : waiting_task_ids) {
- // LOG_TEE("waiting task id %i \n", id_task);
- // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result
- if (result.id_multi == id_task) {
- LOG_VERBOSE("callback_update_multitask", {{"id_task", id_task}});
- callback_update_multitask(id_task, result.id, result);
- continue;
- }
-
- if (result.id == id_task) {
- LOG_VERBOSE("queue_results.push_back", {{"id_task", id_task}});
- queue_results.push_back(result);
- condition_results.notify_all();
- return;
- }
- }
- }
-};
+struct llama_server_context
+{
+ llama_model *model = nullptr;
+ llama_context *ctx = nullptr;
-struct server_context {
- llama_model * model = nullptr;
- llama_context * ctx = nullptr;
+ clip_ctx *clp_ctx = nullptr;
gpt_params params;
llama_batch batch;
- bool clean_kv_cache = true;
- bool add_bos_token = true;
+ bool multimodal = false;
+ bool clean_kv_cache = true;
+ bool all_slots_are_idle = false;
+ bool add_bos_token = true;
- int32_t n_ctx; // total context for all clients / slots
+ int32_t n_ctx; // total context for all clients / slots
// system prompt
bool system_need_update = false;
@@ -646,39 +398,63 @@ struct server_context {
std::string name_assistant;
// slots / clients
- std::vector<server_slot> slots;
+ std::vector<llama_client_slot> slots;
json default_generation_settings_for_props;
- server_queue queue_tasks;
- server_response queue_results;
+ llama_server_queue queue_tasks;
+ llama_server_response queue_results;
- server_metrics metrics;
+ llama_metrics metrics;
- ~server_context() {
- if (ctx) {
+ ~llama_server_context()
+ {
+ if (ctx)
+ {
llama_free(ctx);
ctx = nullptr;
}
-
- if (model) {
+ if (model)
+ {
llama_free_model(model);
model = nullptr;
}
}
- bool load_model(const gpt_params & params_) {
+ bool load_model(const gpt_params &params_)
+ {
params = params_;
+ if (!params.mmproj.empty()) {
+ multimodal = true;
+ LOG_INFO("Multi Modal Mode Enabled", {});
+ clp_ctx = clip_model_load(params.mmproj.c_str(), /*verbosity=*/ 1);
+ if(clp_ctx == nullptr) {
+ LOG_ERROR("unable to load clip model", {{"model", params.mmproj}});
+ return false;
+ }
- // dedicate one sequence to the system prompt
- params.n_parallel += 1;
+ if (params.n_ctx < 2048) { // request larger context for the image embedding
+ params.n_ctx = 2048;
+ }
+ }
std::tie(model, ctx) = llama_init_from_gpt_params(params);
- params.n_parallel -= 1; // but be sneaky about it
- if (model == nullptr) {
+ if (model == nullptr)
+ {
LOG_ERROR("unable to load model", {{"model", params.model}});
return false;
}
+ if (multimodal) {
+ const int n_embd_clip = clip_n_mmproj_embd(clp_ctx);
+ const int n_embd_llm = llama_n_embd(model);
+ if (n_embd_clip != n_embd_llm) {
+ LOG_TEE("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_embd_clip, n_embd_llm);
+ llama_free(ctx);
+ llama_free_model(model);
+ return false;
+ }
+ }
+
n_ctx = llama_n_ctx(ctx);
add_bos_token = llama_should_add_bos_token(model);
@@ -686,28 +462,33 @@ struct server_context {
return true;
}
- bool validate_model_chat_template() const {
+ void validate_model_chat_template(server_params & sparams) {
llama_chat_message chat[] = {{"user", "test"}};
-
- const int res = llama_chat_apply_template(model, nullptr, chat, 1, true, nullptr, 0);
-
- return res > 0;
+ std::vector<char> buf(1);
+ int res = llama_chat_apply_template(model, nullptr, chat, 1, true, buf.data(), buf.size());
+ if (res < 0) {
+ LOG_ERROR("The chat template comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {});
+ sparams.chat_template = "<|im_start|>"; // llama_chat_apply_template only checks if <|im_start|> exist in the template
+ }
}
- void init() {
+ void initialize() {
+ // create slots
+ all_slots_are_idle = true;
+
const int32_t n_ctx_slot = n_ctx / params.n_parallel;
LOG_INFO("initializing slots", {{"n_slots", params.n_parallel}});
-
- for (int i = 0; i < params.n_parallel; i++) {
- server_slot slot;
+ for (int i = 0; i < params.n_parallel; i++)
+ {
+ llama_client_slot slot;
slot.id = i;
slot.n_ctx = n_ctx_slot;
slot.n_predict = params.n_predict;
LOG_INFO("new slot", {
- {"id_slot", slot.id},
+ {"slot_id", slot.id},
{"n_ctx_slot", slot.n_ctx}
});
@@ -715,15 +496,15 @@ struct server_context {
const int ga_w = params.grp_attn_w;
if (ga_n != 1) {
- GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT
- GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT
+ GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT
+ GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT
//GGML_ASSERT(n_ctx_train % ga_w == 0 && "n_ctx_train must be a multiple of ga_w"); // NOLINT
//GGML_ASSERT(n_ctx >= n_ctx_train * ga_n && "n_ctx must be at least n_ctx_train * ga_n"); // NOLINT
LOG_INFO("slot self-extend", {
- {"id_slot", slot.id},
- {"ga_n", ga_n},
- {"ga_w", ga_w}
+ {"slot_id", slot.id},
+ {"ga_n", ga_n},
+ {"ga_w", ga_w}
});
}
@@ -739,18 +520,11 @@ struct server_context {
default_generation_settings_for_props = get_formated_generation(slots.front());
default_generation_settings_for_props["seed"] = -1;
- // the update_slots() logic will always submit a maximum of n_batch tokens
- // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used)
- {
- const int32_t n_batch = llama_n_batch(ctx);
-
- batch = llama_batch_init(n_batch, 0, params.n_parallel);
- }
-
- metrics.init();
+ batch = llama_batch_init(n_ctx, 0, params.n_parallel);
}
- std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const {
+ std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const
+ {
// TODO: currently, we tokenize using special tokens by default
// this is not always correct (see https://github.com/ggerganov/llama.cpp/pull/4160#issuecomment-1824826216)
// but it's better compared to completely ignoring ChatML and other chat templates
@@ -760,30 +534,38 @@ struct server_context {
// or the first element of the json_prompt array is a string.
std::vector<llama_token> prompt_tokens;
- if (json_prompt.is_array()) {
+ if (json_prompt.is_array())
+ {
bool first = true;
- for (const auto & p : json_prompt) {
- if (p.is_string()) {
+ for (const auto& p : json_prompt)
+ {
+ if (p.is_string())
+ {
auto s = p.template get<std::string>();
-
std::vector<llama_token> p;
- if (first) {
+ if (first)
+ {
p = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);
first = false;
- } else {
+ }
+ else
+ {
p = ::llama_tokenize(ctx, s, false, TMP_FORCE_SPECIAL);
}
-
prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());
- } else {
- if (first) {
+ }
+ else
+ {
+ if (first)
+ {
first = false;
}
-
prompt_tokens.push_back(p.template get<llama_token>());
}
}
- } else {
+ }
+ else
+ {
auto s = json_prompt.template get<std::string>();
prompt_tokens = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);
}
@@ -791,18 +573,19 @@ struct server_context {
return prompt_tokens;
}
- server_slot * get_slot(int id) {
+ llama_client_slot* get_slot(int id) {
int64_t t_last = ggml_time_us();
+ llama_client_slot *last_used = nullptr;
- server_slot * last_used = nullptr;
-
- for (server_slot & slot : slots) {
- if (slot.id == id && slot.available()) {
+ for (llama_client_slot & slot : slots)
+ {
+ if (slot.id == id && slot.available())
+ {
return &slot;
}
- // among all available slots, find the one that has been least recently used
- if (slot.available() && slot.t_last_used < t_last) {
+ if (slot.available() && slot.t_last_used < t_last)
+ {
last_used = &slot;
t_last = slot.t_last_used;
}
@@ -811,235 +594,295 @@ struct server_context {
return last_used;
}
- bool launch_slot_with_task(server_slot & slot, const server_task & task) {
+ bool launch_slot_with_data(llama_client_slot* &slot, json data) {
slot_params default_params;
llama_sampling_params default_sparams;
- auto & data = task.data;
if (data.count("__oaicompat") != 0) {
- slot.oaicompat = true;
- slot.oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));
- } else {
- slot.oaicompat = false;
- slot.oaicompat_model = "";
- }
-
- slot.params.stream = json_value(data, "stream", false);
- slot.params.cache_prompt = json_value(data, "cache_prompt", false);
- slot.params.n_predict = json_value(data, "n_predict", default_params.n_predict);
- slot.sparams.top_k = json_value(data, "top_k", default_sparams.top_k);
- slot.sparams.top_p = json_value(data, "top_p", default_sparams.top_p);
- slot.sparams.min_p = json_value(data, "min_p", default_sparams.min_p);
- slot.sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);
- slot.sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);
- slot.sparams.temp = json_value(data, "temperature", default_sparams.temp);
- slot.sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);
- slot.sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);
- slot.sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);
- slot.sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);
- slot.sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);
- slot.sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);
- slot.sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);
- slot.sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);
- slot.sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);
- slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);
- slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep);
- slot.params.seed = json_value(data, "seed", default_params.seed);
- if (data.contains("json_schema") && !data.contains("grammar")) {
- try {
- auto schema = json_value(data, "json_schema", json::object());
- slot.sparams.grammar = json_schema_to_grammar(schema);
- } catch (const std::exception & e) {
- send_error(task, std::string("\"json_schema\": ") + e.what(), ERROR_TYPE_INVALID_REQUEST);
- return false;
- }
+ slot->oaicompat = true;
+ slot->oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));
} else {
- slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar);
- }
- slot.sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);
- slot.sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);
-
- if (slot.params.cache_prompt && slot.ga_n != 1) {
- LOG_WARNING("cache_prompt is not supported with group-attention", {});
- slot.params.cache_prompt = false;
- }
-
- if (slot.n_predict > 0 && slot.params.n_predict > slot.n_predict) {
+ slot->oaicompat = false;
+ slot->oaicompat_model = "";
+ }
+
+ slot->params.stream = json_value(data, "stream", false);
+ slot->params.cache_prompt = json_value(data, "cache_prompt", false);
+ slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict);
+ slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);
+ slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);
+ slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);
+ slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);
+ slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);
+ slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);
+ slot->sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);
+ slot->sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);
+ slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);
+ slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);
+ slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);
+ slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);
+ slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);
+ slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);
+ slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);
+ slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);
+ slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep);
+ slot->params.seed = json_value(data, "seed", default_params.seed);
+ slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar);
+ slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);
+ slot->sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);
+
+ if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) {
// Might be better to reject the request with a 400 ?
LOG_WARNING("Max tokens to predict exceeds server configuration", {
- {"params.n_predict", slot.params.n_predict},
- {"slot.n_predict", slot.n_predict},
+ {"params.n_predict", slot->params.n_predict},
+ {"slot.n_predict", slot->n_predict},
});
- slot.params.n_predict = slot.n_predict;
+ slot->params.n_predict = slot->n_predict;
}
// infill
- slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix);
- slot.params.input_suffix = json_value(data, "input_suffix", default_params.input_suffix);
-
- // get prompt
+ if (data.count("input_prefix") != 0)
{
- const auto & prompt = data.find("prompt");
- if (prompt == data.end()) {
- send_error(task, "Either \"prompt\" or \"messages\" must be provided", ERROR_TYPE_INVALID_REQUEST);
- return false;
- } else {
- slot.prompt = *prompt;
- }
- if (slot.prompt.is_array() && slot.prompt.size() == 0) {
- send_error(task, "\"prompt\" cannot be an empty array", ERROR_TYPE_INVALID_REQUEST);
- return false;
- }
+ slot->params.input_prefix = data["input_prefix"];
}
-
- // penalize user-provided tokens
+ else
{
- slot.sparams.penalty_prompt_tokens.clear();
- slot.sparams.use_penalty_prompt_tokens = false;
-
- const auto & penalty_prompt = data.find("penalty_prompt");
+ slot->params.input_prefix = "";
+ }
- if (penalty_prompt != data.end()) {
- if (penalty_prompt->is_string()) {
- const auto penalty_prompt_string = penalty_prompt->get<std::string>();
- slot.sparams.penalty_prompt_tokens = llama_tokenize(model, penalty_prompt_string, false);
+ if (data.count("input_suffix") != 0)
+ {
+ slot->params.input_suffix = data["input_suffix"];
+ }
+ else
+ {
+ slot->params.input_suffix = "";
+ }
- if (slot.params.n_predict > 0) {
- slot.sparams.penalty_prompt_tokens.reserve(slot.sparams.penalty_prompt_tokens.size() + slot.params.n_predict);
- }
- slot.sparams.use_penalty_prompt_tokens = true;
+ if (data.count("prompt") != 0)
+ {
+ slot->prompt = data["prompt"];
+ }
+ else
+ {
+ slot->prompt = "";
+ }
- LOG_VERBOSE("penalty_prompt_tokens", {
- {"id_slot", slot.id},
- {"tokens", slot.sparams.penalty_prompt_tokens},
- });
+ slot->sparams.penalty_prompt_tokens.clear();
+ slot->sparams.use_penalty_prompt_tokens = false;
+ const auto &penalty_prompt = data.find("penalty_prompt");
+ if (penalty_prompt != data.end())
+ {
+ if (penalty_prompt->is_string())
+ {
+ const auto penalty_prompt_string = penalty_prompt->get<std::string>();
+ auto penalty_tokens = llama_tokenize(model, penalty_prompt_string, false);
+ slot->sparams.penalty_prompt_tokens.swap(penalty_tokens);
+ if (slot->params.n_predict > 0)
+ {
+ slot->sparams.penalty_prompt_tokens.reserve(slot->sparams.penalty_prompt_tokens.size() + slot->params.n_predict);
}
- else if (penalty_prompt->is_array()) {
- const auto n_tokens = penalty_prompt->size();
- slot.sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot.params.n_predict));
-
- const int n_vocab = llama_n_vocab(model);
- for (const auto & penalty_token : *penalty_prompt) {
- if (penalty_token.is_number_integer()) {
- const auto tok = penalty_token.get<llama_token>();
- if (tok >= 0 && tok < n_vocab) {
- slot.sparams.penalty_prompt_tokens.push_back(tok);
- }
+ slot->sparams.use_penalty_prompt_tokens = true;
+ }
+ else if (penalty_prompt->is_array())
+ {
+ const auto n_tokens = penalty_prompt->size();
+ slot->sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot->params.n_predict));
+ const int n_vocab = llama_n_vocab(model);
+ for (const auto &penalty_token : *penalty_prompt)
+ {
+ if (penalty_token.is_number_integer())
+ {
+ const auto tok = penalty_token.get<llama_token>();
+ if (tok >= 0 && tok < n_vocab)
+ {
+ slot->sparams.penalty_prompt_tokens.push_back(tok);
}
}
- slot.sparams.use_penalty_prompt_tokens = true;
-
- LOG_VERBOSE("penalty_prompt_tokens", {
- {"id_slot", slot.id},
- {"tokens", slot.sparams.penalty_prompt_tokens},
- });
}
+ slot->sparams.use_penalty_prompt_tokens = true;
}
}
+ slot->sparams.logit_bias.clear();
+
+ if (json_value(data, "ignore_eos", false))
{
- slot.sparams.logit_bias.clear();
+ slot->sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
+ }
- if (json_value(data, "ignore_eos", false)) {
- slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
- }
+ const auto &logit_bias = data.find("logit_bias");
+ if (logit_bias != data.end() && logit_bias->is_array())
+ {
+ const int n_vocab = llama_n_vocab(model);
+ for (const auto &el : *logit_bias)
+ {
+ if (el.is_array() && el.size() == 2)
+ {
+ float bias;
+ if (el[1].is_number())
+ {
+ bias = el[1].get<float>();
+ }
+ else if (el[1].is_boolean() && !el[1].get<bool>())
+ {
+ bias = -INFINITY;
+ }
+ else
+ {
+ continue;
+ }
- const auto & logit_bias = data.find("logit_bias");
- if (logit_bias != data.end() && logit_bias->is_array()) {
- const int n_vocab = llama_n_vocab(model);
- for (const auto & el : *logit_bias) {
- // TODO: we may want to throw errors here, in case "el" is incorrect
- if (el.is_array() && el.size() == 2) {
- float bias;
- if (el[1].is_number()) {
- bias = el[1].get<float>();
- } else if (el[1].is_boolean() && !el[1].get<bool>()) {
- bias = -INFINITY;
- } else {
- continue;
+ if (el[0].is_number_integer())
+ {
+ llama_token tok = el[0].get<llama_token>();
+ if (tok >= 0 && tok < n_vocab)
+ {
+ slot->sparams.logit_bias[tok] = bias;
}
-
- if (el[0].is_number_integer()) {
- llama_token tok = el[0].get<llama_token>();
- if (tok >= 0 && tok < n_vocab) {
- slot.sparams.logit_bias[tok] = bias;
- }
- } else if (el[0].is_string()) {
- auto toks = llama_tokenize(model, el[0].get<std::string>(), false);
- for (auto tok : toks) {
- slot.sparams.logit_bias[tok] = bias;
- }
+ }
+ else if (el[0].is_string())
+ {
+ auto toks = llama_tokenize(model, el[0].get<std::string>(), false);
+ for (auto tok : toks)
+ {
+ slot->sparams.logit_bias[tok] = bias;
}
}
}
}
}
+ slot->params.antiprompt.clear();
+
+ const auto &stop = data.find("stop");
+ if (stop != data.end() && stop->is_array())
{
- slot.params.antiprompt.clear();
+ for (const auto &word : *stop)
+ {
+ if (!word.empty())
+ {
+ slot->params.antiprompt.push_back(word);
+ }
+ }
+ }
- const auto & stop = data.find("stop");
- if (stop != data.end() && stop->is_array()) {
- for (const auto & word : *stop) {
- if (!word.empty()) {
- slot.params.antiprompt.push_back(word);
- }
+ const auto &samplers_sequence = data.find("samplers");
+ if (samplers_sequence != data.end() && samplers_sequence->is_array())
+ {
+ std::vector<std::string> sampler_names;
+ for (const auto &sampler_name : *samplers_sequence)
+ {
+ if (sampler_name.is_string())
+ {
+ sampler_names.emplace_back(sampler_name);
}
}
+ slot->sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);
+ }
+ else
+ {
+ slot->sparams.samplers_sequence = default_sparams.samplers_sequence;
}
+ if (multimodal)
{
- const auto & samplers_sequence = data.find("samplers");
- if (samplers_sequence != data.end() && samplers_sequence->is_array()) {
- std::vector<std::string> sampler_names;
- for (const auto & sampler_name : *samplers_sequence) {
- if (sampler_name.is_string()) {
- sampler_names.emplace_back(sampler_name);
+ const auto &images_data = data.find("image_data");
+ if (images_data != data.end() && images_data->is_array())
+ {
+ for (const auto &img : *images_data)
+ {
+ const std::vector<uint8_t> image_buffer = base64_decode(img["data"].get<std::string>());
+
+ slot_image img_sl;
+ img_sl.id = img.count("id") != 0 ? img["id"].get<int>() : slot->images.size();
+ img_sl.img_data = clip_image_u8_init();
+ if (!clip_image_load_from_bytes(image_buffer.data(), image_buffer.size(), img_sl.img_data))
+ {
+ LOG_ERROR("failed to load image", {
+ {"slot_id", slot->id},
+ {"img_sl_id", img_sl.id}
+ });
+ return false;
}
+ LOG_VERBOSE("image loaded", {
+ {"slot_id", slot->id},
+ {"img_sl_id", img_sl.id}
+ });
+ img_sl.request_encode_image = true;
+ slot->images.push_back(img_sl);
+ }
+ // process prompt
+ // example: system prompt [img-102] user [img-103] describe [img-134] -> [{id: 102, prefix: 'system prompt '}, {id: 103, prefix: ' user '}, {id: 134, prefix: ' describe '}]}
+ if (slot->images.size() > 0 && !slot->prompt.is_array())
+ {
+ std::string prompt = slot->prompt.get<std::string>();
+ size_t pos = 0, begin_prefix = 0;
+ std::string pattern = "[img-";
+ while ((pos = prompt.find(pattern, pos)) != std::string::npos) {
+ size_t end_prefix = pos;
+ pos += pattern.length();
+ size_t end_pos = prompt.find(']', pos);
+ if (end_pos != std::string::npos)
+ {
+ std::string image_id = prompt.substr(pos, end_pos - pos);
+ try
+ {
+ int img_id = std::stoi(image_id);
+ bool found = false;
+ for (slot_image &img : slot->images)
+ {
+ if (img.id == img_id) {
+ found = true;
+ img.prefix_prompt = prompt.substr(begin_prefix, end_prefix - begin_prefix);
+ begin_prefix = end_pos + 1;
+ break;
+ }
+ }
+ if (!found) {
+ LOG_TEE("ERROR: Image with id: %i, not found.\n", img_id);
+ slot->images.clear();
+ return false;
+ }
+ } catch (const std::invalid_argument& e) {
+ LOG_TEE("Invalid image number id in prompt\n");
+ slot->images.clear();
+ return false;
+ }
+ }
+ }
+ slot->prompt = "";
+ slot->params.input_suffix = prompt.substr(begin_prefix);
+ slot->params.cache_prompt = false; // multimodal doesn't support cache prompt
}
- slot.sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);
- } else {
- slot.sparams.samplers_sequence = default_sparams.samplers_sequence;
}
}
+ if (slot->ctx_sampling != nullptr)
{
- if (slot.ctx_sampling != nullptr) {
- llama_sampling_free(slot.ctx_sampling);
- }
- slot.ctx_sampling = llama_sampling_init(slot.sparams);
- if (slot.ctx_sampling == nullptr) {
- // for now, the only error that may happen here is invalid grammar
- send_error(task, "Failed to parse grammar", ERROR_TYPE_INVALID_REQUEST);
- return false;
- }
- llama_set_rng_seed(ctx, slot.params.seed);
+ llama_sampling_free(slot->ctx_sampling);
}
+ slot->ctx_sampling = llama_sampling_init(slot->sparams);
+ llama_set_rng_seed(ctx, slot->params.seed);
+ slot->command = LOAD_PROMPT;
- slot.command = SLOT_COMMAND_LOAD_PROMPT;
- slot.prompt_tokens.clear();
+ all_slots_are_idle = false;
LOG_INFO("slot is processing task", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
+ {"slot_id", slot->id},
+ {"task_id", slot->task_id},
});
return true;
}
void kv_cache_clear() {
- LOG_VERBOSE("clearing KV cache", {});
-
// clear the entire KV cache
llama_kv_cache_clear(ctx);
clean_kv_cache = false;
}
- void system_prompt_update() {
- LOG_VERBOSE("system prompt update", {
- {"system_prompt", system_prompt},
- });
-
+ void update_system_prompt() {
kv_cache_clear();
system_tokens.clear();
@@ -1048,14 +891,14 @@ struct server_context {
llama_batch_clear(batch);
- for (int i = 0; i < (int)system_tokens.size(); ++i) {
+ for (int i = 0; i < (int)system_tokens.size(); ++i)
+ {
llama_batch_add(batch, system_tokens[i], i, { 0 }, false);
}
- const int32_t n_batch = llama_n_batch(ctx);
-
- for (int32_t i = 0; i < batch.n_tokens; i += n_batch) {
- const int32_t n_tokens = std::min(params.n_batch, batch.n_tokens - i);
+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch)
+ {
+ const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i));
llama_batch batch_view = {
n_tokens,
batch.token + i,
@@ -1066,42 +909,78 @@ struct server_context {
batch.logits + i,
0, 0, 0, // unused
};
-
- if (llama_decode(ctx, batch_view) != 0) {
+ if (llama_decode(ctx, batch_view) != 0)
+ {
LOG_TEE("%s: llama_decode() failed\n", __func__);
return;
}
}
// assign the system KV cache to all parallel sequences
- for (int32_t i = 1; i <= params.n_parallel; ++i) {
- llama_kv_cache_seq_cp(ctx, 0, i, -1, -1);
+ for (int32_t i = 1; i < params.n_parallel; ++i)
+ {
+ llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size());
}
}
+ LOG_TEE("system prompt updated\n");
system_need_update = false;
}
- void system_prompt_set(const json & sys_props) {
+ void notify_system_prompt_changed() {
+ // release all slots
+ for (llama_client_slot &slot : slots)
+ {
+ slot.release();
+ }
+
+ system_need_update = true;
+ }
+
+ void process_system_prompt_data(const json &sys_props) {
system_prompt = sys_props.value("prompt", "");
name_user = sys_props.value("anti_prompt", "");
name_assistant = sys_props.value("assistant_name", "");
- LOG_VERBOSE("system prompt process", {
- {"system_prompt", system_prompt},
- {"name_user", name_user},
- {"name_assistant", name_assistant},
- });
- // release all slots
- for (server_slot & slot : slots) {
- slot.release();
+ notify_system_prompt_changed();
+ }
+
+ static size_t find_stopping_strings(const std::string &text, const size_t last_token_size,
+ const stop_type type, llama_client_slot &slot)
+ {
+ size_t stop_pos = std::string::npos;
+
+ for (const std::string &word : slot.params.antiprompt)
+ {
+ size_t pos;
+ if (type == STOP_FULL)
+ {
+ const size_t tmp = word.size() + last_token_size;
+ const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;
+ pos = text.find(word, from_pos);
+ }
+ else
+ {
+ pos = find_partial_stop_string(word, text);
+ }
+ if (pos != std::string::npos &&
+ (stop_pos == std::string::npos || pos < stop_pos))
+ {
+ if (type == STOP_FULL)
+ {
+ slot.stopped_word = true;
+ slot.stopping_word = word;
+ slot.has_next_token = false;
+ }
+ stop_pos = pos;
+ }
}
- system_need_update = true;
+ return stop_pos;
}
- bool process_token(completion_token_output & result, server_slot & slot) {
+ bool process_token(completion_token_output &result, llama_client_slot &slot) {
// remember which tokens were sampled - used for repetition penalties during sampling
const std::string token_str = llama_token_to_piece(ctx, result.tok);
slot.sampled = result.tok;
@@ -1110,26 +989,34 @@ struct server_context {
slot.generated_text += token_str;
slot.has_next_token = true;
- if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1) {
+ if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1)
+ {
// we can change penalty_prompt_tokens because it is always created from scratch each request
slot.ctx_sampling->params.penalty_prompt_tokens.push_back(result.tok);
}
// check if there is incomplete UTF-8 character at the end
bool incomplete = false;
- for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i) {
+ for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i)
+ {
unsigned char c = slot.generated_text[slot.generated_text.size() - i];
- if ((c & 0xC0) == 0x80) {
+ if ((c & 0xC0) == 0x80)
+ {
// continuation byte: 10xxxxxx
continue;
}
- if ((c & 0xE0) == 0xC0) {
+ if ((c & 0xE0) == 0xC0)
+ {
// 2-byte character: 110xxxxx ...
incomplete = i < 2;
- } else if ((c & 0xF0) == 0xE0) {
+ }
+ else if ((c & 0xF0) == 0xE0)
+ {
// 3-byte character: 1110xxxx ...
incomplete = i < 3;
- } else if ((c & 0xF8) == 0xF0) {
+ }
+ else if ((c & 0xF8) == 0xF0)
+ {
// 4-byte character: 11110xxx ...
incomplete = i < 4;
}
@@ -1137,196 +1024,210 @@ struct server_context {
break;
}
- if (!incomplete) {
- size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());
-
+ if (!incomplete)
+ {
+ size_t pos = std::min(slot.sent_count, slot.generated_text.size());
const std::string str_test = slot.generated_text.substr(pos);
bool is_stop_full = false;
-
- size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_FULL);
- if (stop_pos != std::string::npos) {
+ size_t stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_FULL, slot);
+ if (stop_pos != std::string::npos)
+ {
is_stop_full = true;
slot.generated_text.erase(
slot.generated_text.begin() + pos + stop_pos,
slot.generated_text.end());
- pos = std::min(slot.n_sent_text, slot.generated_text.size());
- } else {
+ pos = std::min(slot.sent_count, slot.generated_text.size());
+ }
+ else
+ {
is_stop_full = false;
- stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_PARTIAL);
+ stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_PARTIAL, slot);
}
// check if there is any token to predict
- if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0)) {
+ if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0))
+ {
// no send the stop word in the response
result.text_to_send = slot.generated_text.substr(pos, std::string::npos);
- slot.n_sent_text += result.text_to_send.size();
+ slot.sent_count += result.text_to_send.size();
// add the token to slot queue and cache
}
-
slot.add_token_string(result);
- if (slot.params.stream) {
+ if (slot.params.stream)
+ {
send_partial_response(slot, result);
}
}
- if (incomplete) {
+ if (incomplete)
+ {
slot.has_next_token = true;
}
// check the limits
- if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params)) {
- slot.stopped_limit = true;
+ if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params))
+ {
+ slot.stopped_limit = true;
slot.has_next_token = false;
-
- LOG_VERBOSE("stopped by limit", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
- {"n_decoded", slot.n_decoded},
- {"n_predict", slot.params.n_predict},
- });
}
- if (result.tok == llama_token_eos(model)) {
- slot.stopped_eos = true;
+ if (!slot.cache_tokens.empty() && result.tok == llama_token_eos(model))
+ {
+ slot.stopped_eos = true;
slot.has_next_token = false;
-
LOG_VERBOSE("eos token found", {});
}
LOG_VERBOSE("next token", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
- {"token", result.tok},
- {"token_text", tokens_to_output_formatted_string(ctx, result.tok)},
- {"has_next_token", slot.has_next_token},
- {"n_remain", slot.n_remaining},
- {"n_decoded", slot.n_decoded},
- {"stopped_eos", slot.stopped_eos},
- {"stopped_word", slot.stopped_word},
- {"stopped_limit", slot.stopped_limit},
- {"stopping_word", slot.stopping_word},
- });
+ {"token", result.tok},
+ {"token_text", tokens_to_output_formatted_string(ctx, result.tok)},
+ {"has_next_token", slot.has_next_token},
+ {"n_remain", slot.n_remaining},
+ {"num_tokens_predicted", slot.n_decoded},
+ {"stopped_eos", slot.stopped_eos},
+ {"stopped_word", slot.stopped_word},
+ {"stopped_limit", slot.stopped_limit},
+ {"stopping_word", slot.stopping_word},
+ });
return slot.has_next_token; // continue
}
- json get_formated_generation(const server_slot & slot) const {
- const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));
- const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && eos_bias->second < 0.0f && std::isinf(eos_bias->second);
+ bool process_images(llama_client_slot &slot) const
+ {
+ for (slot_image &img : slot.images)
+ {
+ if (!img.request_encode_image)
+ {
+ continue;
+ }
+
+ if (!llava_image_embed_make_with_clip_img(clp_ctx, params.n_threads, img.img_data, &img.image_embedding, &img.image_tokens)) {
+ LOG_TEE("Error processing the given image");
+ return false;
+ }
+
+
+ img.request_encode_image = false;
+ }
+
+ return slot.images.size() > 0;
+ }
+ void send_error(task_server& task, const std::string &error)
+ {
+ LOG_TEE("task %i - error: %s\n", task.id, error.c_str());
+ task_result res;
+ res.id = task.id;
+ res.multitask_id = task.multitask_id;
+ res.stop = false;
+ res.error = true;
+ res.result_json = { { "content", error } };
+ queue_results.send(res);
+ }
+
+ json get_formated_generation(llama_client_slot &slot)
+ {
+ const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));
+ const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() &&
+ eos_bias->second < 0.0f && std::isinf(eos_bias->second);
std::vector<std::string> samplers_sequence;
- samplers_sequence.reserve(slot.sparams.samplers_sequence.size());
- for (const auto & sampler_type : slot.sparams.samplers_sequence) {
+ for (const auto &sampler_type : slot.sparams.samplers_sequence)
+ {
samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type));
}
return json {
- {"n_ctx", slot.n_ctx},
- {"n_predict", slot.n_predict},
- {"model", params.model_alias},
- {"seed", slot.params.seed},
- {"temperature", slot.sparams.temp},
- {"dynatemp_range", slot.sparams.dynatemp_range},
- {"dynatemp_exponent", slot.sparams.dynatemp_exponent},
- {"top_k", slot.sparams.top_k},
- {"top_p", slot.sparams.top_p},
- {"min_p", slot.sparams.min_p},
- {"tfs_z", slot.sparams.tfs_z},
- {"typical_p", slot.sparams.typical_p},
- {"repeat_last_n", slot.sparams.penalty_last_n},
- {"repeat_penalty", slot.sparams.penalty_repeat},
- {"presence_penalty", slot.sparams.penalty_present},
- {"frequency_penalty", slot.sparams.penalty_freq},
- {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},
+ {"n_ctx", slot.n_ctx},
+ {"n_predict", slot.n_predict},
+ {"model", params.model_alias},
+ {"seed", slot.params.seed},
+ {"temperature", slot.sparams.temp},
+ {"dynatemp_range", slot.sparams.dynatemp_range},
+ {"dynatemp_exponent", slot.sparams.dynatemp_exponent},
+ {"top_k", slot.sparams.top_k},
+ {"top_p", slot.sparams.top_p},
+ {"min_p", slot.sparams.min_p},
+ {"tfs_z", slot.sparams.tfs_z},
+ {"typical_p", slot.sparams.typical_p},
+ {"repeat_last_n", slot.sparams.penalty_last_n},
+ {"repeat_penalty", slot.sparams.penalty_repeat},
+ {"presence_penalty", slot.sparams.penalty_present},
+ {"frequency_penalty", slot.sparams.penalty_freq},
+ {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},
{"use_penalty_prompt_tokens", slot.sparams.use_penalty_prompt_tokens},
- {"mirostat", slot.sparams.mirostat},
- {"mirostat_tau", slot.sparams.mirostat_tau},
- {"mirostat_eta", slot.sparams.mirostat_eta},
- {"penalize_nl", slot.sparams.penalize_nl},
- {"stop", slot.params.antiprompt},
- {"n_predict", slot.params.n_predict}, // TODO: fix duplicate key n_predict
- {"n_keep", params.n_keep},
- {"ignore_eos", ignore_eos},
- {"stream", slot.params.stream},
- {"logit_bias", slot.sparams.logit_bias},
- {"n_probs", slot.sparams.n_probs},
- {"min_keep", slot.sparams.min_keep},
- {"grammar", slot.sparams.grammar},
- {"samplers", samplers_sequence}
+ {"mirostat", slot.sparams.mirostat},
+ {"mirostat_tau", slot.sparams.mirostat_tau},
+ {"mirostat_eta", slot.sparams.mirostat_eta},
+ {"penalize_nl", slot.sparams.penalize_nl},
+ {"stop", slot.params.antiprompt},
+ {"n_predict", slot.params.n_predict},
+ {"n_keep", params.n_keep},
+ {"ignore_eos", ignore_eos},
+ {"stream", slot.params.stream},
+ {"logit_bias", slot.sparams.logit_bias},
+ {"n_probs", slot.sparams.n_probs},
+ {"min_keep", slot.sparams.min_keep},
+ {"grammar", slot.sparams.grammar},
+ {"samplers", samplers_sequence}
};
}
- void send_error(const server_task & task, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {
- send_error(task.id, task.id_multi, error, type);
- }
-
- void send_error(const server_slot & slot, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {
- send_error(slot.id_task, slot.id_multi, error, type);
- }
-
- void send_error(const int id_task, const int id_multi, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {
- LOG_TEE("task %i - error: %s\n", id_task, error.c_str());
-
- server_task_result res;
- res.id = id_task;
- res.id_multi = id_multi;
- res.stop = false;
- res.error = true;
- res.data = format_error_response(error, type);
-
- queue_results.send(res);
- }
+ void send_partial_response(llama_client_slot &slot, completion_token_output tkn)
+ {
+ task_result res;
+ res.id = slot.task_id;
+ res.multitask_id = slot.multitask_id;
+ res.error = false;
+ res.stop = false;
- void send_partial_response(server_slot & slot, completion_token_output tkn) {
- server_task_result res;
- res.id = slot.id_task;
- res.id_multi = slot.id_multi;
- res.error = false;
- res.stop = false;
- res.data = json {
+ res.result_json = json
+ {
{"content", tkn.text_to_send},
{"stop", false},
- {"id_slot", slot.id},
- {"multimodal", false}
+ {"slot_id", slot.id},
+ {"multimodal", multimodal}
};
- if (slot.sparams.n_probs > 0) {
+ if (slot.sparams.n_probs > 0)
+ {
+ std::vector<completion_token_output> probs_output = {};
const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);
- const size_t probs_pos = std::min(slot.n_sent_token_probs, slot.generated_token_probs.size());
- const size_t probs_stop_pos = std::min(slot.n_sent_token_probs + to_send_toks.size(), slot.generated_token_probs.size());
-
- std::vector<completion_token_output> probs_output;
- if (probs_pos < probs_stop_pos) {
- probs_output = std::vector<completion_token_output>(
- slot.generated_token_probs.begin() + probs_pos,
- slot.generated_token_probs.begin() + probs_stop_pos);
+ size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());
+ size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size());
+ if (probs_pos < probs_stop_pos)
+ {
+ probs_output = std::vector<completion_token_output>(slot.generated_token_probs.begin() + probs_pos, slot.generated_token_probs.begin() + probs_stop_pos);
}
- slot.n_sent_token_probs = probs_stop_pos;
-
- res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs_output);
+ slot.sent_token_probs_index = probs_stop_pos;
+ res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs_output);
}
- if (slot.oaicompat) {
- res.data["oaicompat_token_ctr"] = slot.n_decoded;
- res.data["model"] = slot.oaicompat_model;
+ if (slot.oaicompat)
+ {
+ res.result_json["oaicompat_token_ctr"] = slot.n_decoded;
+ res.result_json["model"] = slot.oaicompat_model;
}
queue_results.send(res);
}
- void send_final_response(const server_slot & slot) {
- server_task_result res;
- res.id = slot.id_task;
- res.id_multi = slot.id_multi;
- res.error = false;
- res.stop = true;
- res.data = json {
+ void send_final_response(llama_client_slot &slot)
+ {
+ task_result res;
+ res.id = slot.task_id;
+ res.multitask_id = slot.multitask_id;
+ res.error = false;
+ res.stop = true;
+
+ res.result_json = json
+ {
{"content", !slot.params.stream ? slot.generated_text : ""},
- {"id_slot", slot.id},
+ {"slot_id", slot.id},
{"stop", true},
{"model", params.model_alias},
{"tokens_predicted", slot.n_decoded},
- {"tokens_evaluated", slot.n_prompt_tokens},
+ {"tokens_evaluated", slot.num_prompt_tokens},
{"generation_settings", get_formated_generation(slot)},
{"prompt", slot.prompt},
{"truncated", slot.truncated},
@@ -1338,91 +1239,80 @@ struct server_context {
{"timings", slot.get_formated_timings()}
};
- if (slot.sparams.n_probs > 0) {
- std::vector<completion_token_output> probs;
- if (!slot.params.stream && slot.stopped_word) {
+ if (slot.sparams.n_probs > 0)
+ {
+ std::vector<completion_token_output> probs = {};
+ if (!slot.params.stream && slot.stopped_word)
+ {
const std::vector<llama_token> stop_word_toks = llama_tokenize(ctx, slot.stopping_word, false);
-
- probs = std::vector<completion_token_output>(
- slot.generated_token_probs.begin(),
- slot.generated_token_probs.end() - stop_word_toks.size());
- } else {
+ probs = std::vector<completion_token_output>(slot.generated_token_probs.begin(), slot.generated_token_probs.end() - stop_word_toks.size());
+ }
+ else
+ {
probs = std::vector<completion_token_output>(
- slot.generated_token_probs.begin(),
- slot.generated_token_probs.end());
+ slot.generated_token_probs.begin(),
+ slot.generated_token_probs.end());
}
-
- res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs);
+ res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);
}
- if (slot.oaicompat) {
- res.data["oaicompat_token_ctr"] = slot.n_decoded;
- res.data["model"] = slot.oaicompat_model;
+ if (slot.oaicompat)
+ {
+ res.result_json["oaicompat_token_ctr"] = slot.n_decoded;
+ res.result_json["model"] = slot.oaicompat_model;
}
queue_results.send(res);
}
- void send_embedding(const server_slot & slot, const llama_batch & batch) {
- server_task_result res;
- res.id = slot.id_task;
- res.id_multi = slot.id_multi;
- res.error = false;
- res.stop = true;
+ void send_embedding(llama_client_slot &slot)
+ {
+ task_result res;
+ res.id = slot.task_id;
+ res.multitask_id = slot.multitask_id;
+ res.error = false;
+ res.stop = true;
const int n_embd = llama_n_embd(model);
-
- std::vector<float> embd_res(n_embd, 0.0f);
-
- for (int i = 0; i < batch.n_tokens; ++i) {
- if (!batch.logits[i] || batch.seq_id[i][0] != slot.id + 1) {
- continue;
- }
-
- const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]);
- if (embd == NULL) {
- embd = llama_get_embeddings_ith(ctx, i);
- }
-
- if (embd == NULL) {
- LOG_ERROR("failed to get embeddings", {
- {"token", batch.token [i]},
- {"seq_id", batch.seq_id[i][0]}
- });
-
- res.data = json {
- {"embedding", std::vector<float>(n_embd, 0.0f)},
- };
-
- continue;
- }
-
- llama_embd_normalize(embd, embd_res.data(), n_embd);
-
- res.data = json {
- {"embedding", embd_res},
+ if (!params.embedding)
+ {
+ LOG_WARNING("embedding disabled", {
+ {"params.embedding", params.embedding},
+ });
+ res.result_json = json
+ {
+ {"embedding", std::vector<float>(n_embd, 0.0f)},
+ };
+ }
+ else
+ {
+ const float *data = llama_get_embeddings(ctx);
+ std::vector<float> embedding(data, data + n_embd);
+ res.result_json = json
+ {
+ {"embedding", embedding },
};
}
-
queue_results.send(res);
}
- void request_completion(int id_task, int id_multi, json data, bool infill, bool embedding) {
- server_task task;
- task.id = id_task;
- task.id_multi = id_multi;
- task.id_target = 0;
- task.data = std::move(data);
- task.infill = infill;
- task.embedding = embedding;
- task.type = SERVER_TASK_TYPE_COMPLETION;
+ void request_completion(int task_id, json data, bool infill, bool embedding, int multitask_id)
+ {
+ task_server task;
+ task.id = task_id;
+ task.target_id = 0;
+ task.data = std::move(data);
+ task.infill_mode = infill;
+ task.embedding_mode = embedding;
+ task.type = TASK_TYPE_COMPLETION;
+ task.multitask_id = multitask_id;
// when a completion task's prompt array is not a singleton, we split it into multiple requests
// otherwise, it's a single-prompt task, we actually queue it
// if there's numbers in the prompt array it will be treated as an array of tokens
if (task.data.count("prompt") != 0 && task.data.at("prompt").size() > 1) {
bool numbers = false;
- for (const auto & e : task.data.at("prompt")) {
+ for (const auto& e : task.data.at("prompt")) {
if (e.is_number()) {
numbers = true;
break;
@@ -1437,23 +1327,96 @@ struct server_context {
if (numbers) {
queue_tasks.post(task);
} else {
- split_multiprompt_task(id_task, task);
+ split_multiprompt_task(task_id, task);
}
} else {
+ // an empty prompt can make slot become buggy
+ if (task.data.contains("prompt") && task.data["prompt"].is_string() && task.data["prompt"].get<std::string>().empty()) {
+ task.data["prompt"] = " "; // add a space so that we have one token
+ }
queue_tasks.post(task);
}
}
- void request_cancel(int id_task) {
- server_task task;
- task.type = SERVER_TASK_TYPE_CANCEL;
- task.id_target = id_task;
+ // for multiple images processing
+ bool ingest_images(llama_client_slot &slot, int n_batch)
+ {
+ int image_idx = 0;
+
+ while (image_idx < (int) slot.images.size())
+ {
+ slot_image &img = slot.images[image_idx];
+
+ // process prefix prompt
+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)
+ {
+ const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));
+ llama_batch batch_view = {
+ n_tokens,
+ batch.token + i,
+ nullptr,
+ batch.pos + i,
+ batch.n_seq_id + i,
+ batch.seq_id + i,
+ batch.logits + i,
+ 0, 0, 0, // unused
+ };
+ if (llama_decode(ctx, batch_view))
+ {
+ LOG_TEE("%s : failed to eval\n", __func__);
+ return false;
+ }
+ }
+
+ // process image with llm
+ for (int i = 0; i < img.image_tokens; i += n_batch)
+ {
+ int n_eval = img.image_tokens - i;
+ if (n_eval > n_batch)
+ {
+ n_eval = n_batch;
+ }
+
+ const int n_embd = llama_n_embd(model);
+ llama_batch batch_img = { n_eval, nullptr, (img.image_embedding + i * n_embd), nullptr, nullptr, nullptr, nullptr, slot.n_past, 1, 0, };
+ if (llama_decode(ctx, batch_img))
+ {
+ LOG_TEE("%s : failed to eval image\n", __func__);
+ return false;
+ }
+ slot.n_past += n_eval;
+ }
+ image_idx++;
+
+ llama_batch_clear(batch);
+
+ // append prefix of next image
+ const auto json_prompt = (image_idx >= (int) slot.images.size()) ?
+ slot.params.input_suffix : // no more images, then process suffix prompt
+ (json)(slot.images[image_idx].prefix_prompt);
+ std::vector<llama_token> append_tokens = tokenize(json_prompt, false); // has next image
+ for (int i = 0; i < (int) append_tokens.size(); ++i)
+ {
+ llama_batch_add(batch, append_tokens[i], system_tokens.size() + slot.n_past, { slot.id }, true);
+ slot.n_past += 1;
+ }
+ }
+
+ return true;
+ }
+
+ void request_cancel(int task_id)
+ {
+ task_server task;
+ task.type = TASK_TYPE_CANCEL;
+ task.target_id = task_id;
queue_tasks.post(task);
}
- void split_multiprompt_task(int id_multi, const server_task & multiprompt_task) {
- const int prompt_count = multiprompt_task.data.at("prompt").size();
+ void split_multiprompt_task(int multitask_id, task_server& multiprompt_task)
+ {
+ int prompt_count = multiprompt_task.data.at("prompt").size();
if (prompt_count <= 1) {
send_error(multiprompt_task, "error while handling multiple prompts");
return;
@@ -1461,239 +1424,208 @@ struct server_context {
// generate all the ID for subtask
std::vector<int> subtask_ids(prompt_count);
- for (int i = 0; i < prompt_count; i++) {
+ for (int i = 0; i < prompt_count; i++)
+ {
subtask_ids[i] = queue_tasks.get_new_id();
}
// queue up the multitask so we can track its subtask progression
- queue_tasks.add_multitask(id_multi, subtask_ids);
+ queue_tasks.add_multitask(multitask_id, subtask_ids);
// add subtasks
- for (int i = 0; i < prompt_count; i++) {
+ for (int i = 0; i < prompt_count; i++)
+ {
json subtask_data = multiprompt_task.data;
subtask_data["prompt"] = subtask_data["prompt"][i];
// subtasks inherit everything else (infill mode, embedding mode, etc.)
- request_completion(subtask_ids[i], id_multi, subtask_data, multiprompt_task.infill, multiprompt_task.embedding);
+ request_completion(subtask_ids[i], subtask_data, multiprompt_task.infill_mode, multiprompt_task.embedding_mode, multitask_id);
}
}
- void process_single_task(const server_task & task) {
- switch (task.type) {
- case SERVER_TASK_TYPE_COMPLETION:
+ void process_single_task(task_server& task)
+ {
+ switch (task.type)
+ {
+ case TASK_TYPE_COMPLETION: {
+ llama_client_slot *slot = get_slot(json_value(task.data, "slot_id", -1));
+ if (slot == nullptr)
+ {
+ // if no slot is available, we defer this task for processing later
+ LOG_VERBOSE("no slot is available", {{"task_id", task.id}});
+ queue_tasks.defer(task);
+ break;
+ }
+
+ if (task.data.contains("system_prompt"))
{
- server_slot * slot = get_slot(json_value(task.data, "id_slot", -1));
- if (slot == nullptr) {
- // if no slot is available, we defer this task for processing later
- LOG_VERBOSE("no slot is available", {{"id_task", task.id}});
- queue_tasks.defer(task);
+ if (!all_slots_are_idle) {
+ send_error(task, "system prompt can only be updated when all slots are idle");
break;
}
+ process_system_prompt_data(task.data["system_prompt"]);
- if (task.data.contains("system_prompt")) {
- system_prompt_set(task.data["system_prompt"]);
-
- for (server_slot & slot : slots) {
- slot.n_past = 0;
- slot.n_past_se = 0;
- }
+ // reset cache_tokens for all slots
+ for (llama_client_slot &slot : slots)
+ {
+ slot.cache_tokens.clear();
+ slot.n_past = 0;
+ slot.n_past_se = 0;
}
+ }
- slot->reset();
+ slot->reset();
- slot->id_task = task.id;
- slot->id_multi = task.id_multi;
- slot->infill = task.infill;
- slot->embedding = task.embedding;
+ slot->infill = task.infill_mode;
+ slot->embedding = task.embedding_mode;
+ slot->task_id = task.id;
+ slot->multitask_id = task.multitask_id;
- if (!launch_slot_with_task(*slot, task)) {
- LOG_ERROR("error while launching slot", task.data);
- break;
- }
- } break;
- case SERVER_TASK_TYPE_CANCEL:
+ if (!launch_slot_with_data(slot, task.data))
{
- // release slot linked with the task id
- for (auto & slot : slots) {
- if (slot.id_task == task.id_target) {
- slot.release();
- break;
- }
- }
- } break;
- case SERVER_TASK_TYPE_NEXT_RESPONSE:
- {
- // do nothing
- } break;
- case SERVER_TASK_TYPE_METRICS:
+ // send error result
+ send_error(task, "internal_error");
+ break;
+ }
+ } break;
+ case TASK_TYPE_CANCEL: { // release slot linked with the task id
+ for (auto & slot : slots)
{
- json slots_data = json::array();
-
- int n_idle_slots = 0;
- int n_processing_slots = 0;
-
- for (server_slot & slot : slots) {
- json slot_data = get_formated_generation(slot);
- slot_data["id"] = slot.id;
- slot_data["id_task"] = slot.id_task;
- slot_data["state"] = slot.state;
- slot_data["prompt"] = slot.prompt;
- slot_data["next_token"] = {
+ if (slot.task_id == task.target_id)
+ {
+ slot.release();
+ break;
+ }
+ }
+ } break;
+ case TASK_TYPE_NEXT_RESPONSE: {
+ // do nothing
+ } break;
+ case TASK_TYPE_METRICS: {
+ json slots_data = json::array();
+ int n_idle_slots = 0;
+ int n_processing_slots = 0;
+
+ for (llama_client_slot &slot: slots) {
+ json slot_data = get_formated_generation(slot);
+ slot_data["id"] = slot.id;
+ slot_data["task_id"] = slot.task_id;
+ slot_data["state"] = slot.state;
+ slot_data["prompt"] = slot.prompt;
+ slot_data["next_token"] = {
{"has_next_token", slot.has_next_token},
- {"n_remain", slot.n_remaining},
- {"n_decoded", slot.n_decoded},
- {"stopped_eos", slot.stopped_eos},
- {"stopped_word", slot.stopped_word},
- {"stopped_limit", slot.stopped_limit},
- {"stopping_word", slot.stopping_word},
- };
-
- if (slot_data["state"] == SLOT_STATE_IDLE) {
- n_idle_slots++;
- } else {
- n_processing_slots++;
- }
-
- slots_data.push_back(slot_data);
+ {"n_remain", slot.n_remaining},
+ {"num_tokens_predicted", slot.n_decoded},
+ {"stopped_eos", slot.stopped_eos},
+ {"stopped_word", slot.stopped_word},
+ {"stopped_limit", slot.stopped_limit},
+ {"stopping_word", slot.stopping_word},
+ };
+ if (slot_data["state"] == IDLE) {
+ n_idle_slots++;
+ } else {
+ n_processing_slots++;
}
- LOG_INFO("slot data", {
- {"id_task", task.id},
- {"n_idle_slots", n_idle_slots},
- {"n_processing_slots", n_processing_slots}
- });
-
- LOG_VERBOSE("slot data", {
- {"id_task", task.id},
- {"n_idle_slots", n_idle_slots},
- {"n_processing_slots", n_processing_slots},
- {"slots", slots_data}
- });
-
- server_task_result res;
- res.id = task.id;
- res.id_multi = task.id_multi;
- res.stop = true;
- res.error = false;
- res.data = {
+ slots_data.push_back(slot_data);
+ }
+ LOG_INFO("slot data", {
+ {"task_id", task.id},
+ {"n_idle_slots", n_idle_slots},
+ {"n_processing_slots", n_processing_slots}
+ });
+ LOG_VERBOSE("slot data", {
+ {"task_id", task.id},
+ {"n_idle_slots", n_idle_slots},
+ {"n_processing_slots", n_processing_slots},
+ {"slots", slots_data}
+ });
+ task_result res;
+ res.id = task.id;
+ res.multitask_id = task.multitask_id;
+ res.stop = true;
+ res.error = false;
+ res.result_json = {
{ "idle", n_idle_slots },
{ "processing", n_processing_slots },
{ "deferred", queue_tasks.queue_tasks_deferred.size() },
- { "t_start", metrics.t_start},
{ "n_prompt_tokens_processed_total", metrics.n_prompt_tokens_processed_total},
- { "t_tokens_generation_total", metrics.t_tokens_generation_total},
{ "n_tokens_predicted_total", metrics.n_tokens_predicted_total},
- { "t_prompt_processing_total", metrics.t_prompt_processing_total},
{ "n_prompt_tokens_processed", metrics.n_prompt_tokens_processed},
{ "t_prompt_processing", metrics.t_prompt_processing},
{ "n_tokens_predicted", metrics.n_tokens_predicted},
{ "t_tokens_generation", metrics.t_tokens_generation},
- { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)},
- { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)},
-
- { "slots", slots_data },
- };
+ { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)},
+ { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)},
- if (json_value(task.data, "reset_bucket", false)) {
- metrics.reset_bucket();
- }
- queue_results.send(res);
- } break;
+ { "slots", slots_data },
+ };
+ metrics.reset_bucket();
+ queue_results.send(res);
+ } break;
}
}
- void on_finish_multitask(const server_task_multi & multitask) {
+ void on_finish_multitask(task_multi& multitask)
+ {
// all subtasks done == multitask is done
- server_task_result result;
- result.id = multitask.id;
- result.stop = true;
+ task_result result;
+ result.id = multitask.id;
+ result.stop = true;
result.error = false;
// collect json results into one json result
std::vector<json> result_jsons;
- for (const auto & subres : multitask.results) {
- result_jsons.push_back(subres.data);
+ for (auto& subres : multitask.results)
+ {
+ result_jsons.push_back(subres.result_json);
result.error = result.error && subres.error;
}
- result.data = json {
- { "results", result_jsons }
- };
-
+ result.result_json = json{ { "results", result_jsons } };
queue_results.send(result);
}
- void update_slots() {
- if (system_need_update) {
- system_prompt_update();
+ bool update_slots() {
+ if (system_need_update)
+ {
+ LOG_INFO("updating system prompt", {});
+ update_system_prompt();
}
- // release slots
- for (auto & slot : slots) {
- if (slot.command == SLOT_COMMAND_RELEASE) {
- slot.state = SLOT_STATE_IDLE;
- slot.command = SLOT_COMMAND_NONE;
- slot.t_last_used = ggml_time_us();
-
- LOG_INFO("slot released", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
- {"n_ctx", n_ctx},
- {"n_past", slot.n_past},
- {"n_system_tokens", system_tokens.size()},
- {"n_cache_tokens", slot.cache_tokens.size()},
- {"truncated", slot.truncated}
- });
-
- queue_tasks.notify_slot_changed();
- }
- }
+ llama_batch_clear(batch);
- // check if all slots are idle
+ if (all_slots_are_idle)
{
- bool all_idle = true;
-
- for (auto & slot : slots) {
- if (slot.state != SLOT_STATE_IDLE || slot.command != SLOT_COMMAND_NONE) {
- all_idle = false;
- break;
- }
- }
-
- if (all_idle) {
- LOG_INFO("all slots are idle", {});
- if (system_prompt.empty() && clean_kv_cache) {
- kv_cache_clear();
- }
-
- return;
+ if (system_prompt.empty() && clean_kv_cache)
+ {
+ LOG_INFO("all slots are idle and system prompt is empty, clear the KV cache", {});
+ kv_cache_clear();
}
+ return true;
}
- {
- LOG_VERBOSE("posting NEXT_RESPONSE", {});
-
- server_task task;
- task.type = SERVER_TASK_TYPE_NEXT_RESPONSE;
- task.id_target = -1;
-
- queue_tasks.post(task);
- }
+ LOG_VERBOSE("posting NEXT_RESPONSE", {});
+ task_server task;
+ task.type = TASK_TYPE_NEXT_RESPONSE;
+ task.target_id = -1;
+ queue_tasks.post(task);
- // apply context-shift if needed
- // TODO: simplify and improve
- for (server_slot & slot : slots) {
- if (slot.ga_n == 1) {
- if (slot.is_processing() && (int) system_tokens.size() + slot.n_past >= slot.n_ctx - 1) {
+ for (llama_client_slot &slot : slots)
+ {
+ if (slot.ga_n == 1)
+ {
+ if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx)
+ {
// Shift context
const int n_keep = slot.params.n_keep + add_bos_token;
const int n_left = (int) system_tokens.size() + slot.n_past - n_keep;
const int n_discard = n_left / 2;
LOG_INFO("slot context shift", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
+ {"slot_id", slot.id},
+ {"task_id", slot.task_id},
{"n_keep", n_keep},
{"n_left", n_left},
{"n_discard", n_discard},
@@ -1702,18 +1634,16 @@ struct server_context {
{"n_system_tokens", system_tokens.size()},
{"n_cache_tokens", slot.cache_tokens.size()}
});
+ llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard);
+ llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);
- llama_kv_cache_seq_rm (ctx, slot.id + 1, n_keep , n_keep + n_discard);
- llama_kv_cache_seq_add(ctx, slot.id + 1, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);
-
- if (slot.params.cache_prompt) {
- for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) {
- slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];
- }
-
- slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard);
+ for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++)
+ {
+ slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];
}
+ slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard);
+
slot.n_past -= n_discard;
slot.truncated = true;
@@ -1721,12 +1651,33 @@ struct server_context {
}
}
- // start populating the batch for this iteration
- llama_batch_clear(batch);
+ // decode any currently ongoing sequences
+ LOG_VERBOSE("decoding ongoing sequences", {});
+ for (auto & slot : slots)
+ {
+ // release the slot
+ if (slot.command == RELEASE)
+ {
+ slot.state = IDLE;
+ slot.command = NONE;
+ slot.t_last_used = ggml_time_us();
+
+ LOG_INFO("slot released", {
+ {"slot_id", slot.id},
+ {"task_id", slot.task_id},
+ {"n_ctx", n_ctx},
+ {"n_past", slot.n_past},
+ {"n_system_tokens", system_tokens.size()},
+ {"n_cache_tokens", slot.cache_tokens.size()},
+ {"truncated", slot.truncated}
+ });
+ queue_tasks.notify_slot_changed();
+
+ continue;
+ }
- // frist, add sampled tokens from any ongoing sequences
- for (auto & slot : slots) {
- if (slot.state == SLOT_STATE_IDLE) {
+ if (slot.state == IDLE)
+ {
continue;
}
@@ -1736,217 +1687,187 @@ struct server_context {
// TODO: we always have to take into account the "system_tokens"
// this is not great and needs to be improved somehow
- llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id + 1 }, true);
-
+ llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true);
slot.n_past += 1;
-
- if (slot.params.cache_prompt) {
- slot.cache_tokens.push_back(slot.sampled);
- }
-
- LOG_VERBOSE("slot decode token", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
- {"n_ctx", n_ctx},
- {"n_past", slot.n_past},
- {"n_system_tokens", system_tokens.size()},
- {"n_cache_tokens", slot.cache_tokens.size()},
- {"truncated", slot.truncated}
- });
}
// process in chunks of params.n_batch
- int32_t n_batch = llama_n_batch(ctx);
- int32_t n_ubatch = llama_n_ubatch(ctx);
-
- // next, batch any pending prompts without exceeding n_batch
- if (params.cont_batching || batch.n_tokens == 0) {
- for (auto & slot : slots) {
- // this slot still has a prompt to be processed
- if (slot.state == SLOT_STATE_IDLE && slot.command == SLOT_COMMAND_LOAD_PROMPT) {
- auto & prompt_tokens = slot.prompt_tokens;
-
- // we haven't tokenized the prompt yet - do it now:
- if (prompt_tokens.empty()) {
- LOG_VERBOSE("tokenizing prompt", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task}
- });
-
- slot.t_start_process_prompt = ggml_time_us();
- slot.t_start_generation = 0;
+ int32_t n_batch = params.n_batch;
- if (slot.infill) {
- bool suff_rm_leading_spc = true;
- if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1) {
- params.input_suffix.erase(0, 1);
- suff_rm_leading_spc = false;
- }
-
- auto prefix_tokens = tokenize(slot.params.input_prefix, false);
- auto suffix_tokens = tokenize(slot.params.input_suffix, false);
+ // assign workload to the slots
+ if (params.cont_batching || batch.n_tokens == 0)
+ {
+ for (auto & slot : slots)
+ {
+ const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get<std::string>().empty()) || !slot.images.empty();
- const int space_token = 29871; // TODO: this should not be hardcoded
- if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {
- suffix_tokens.erase(suffix_tokens.begin());
- }
+ // empty prompt passed -> release the slot and send empty response
+ // note: infill mode allows empty prompt
+ if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt && !slot.infill)
+ {
+ slot.release();
+ slot.print_timings();
+ send_final_response(slot);
+ continue;
+ }
- prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));
- prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS
- prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));
- prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());
- prefix_tokens.push_back(llama_token_middle(model));
- prompt_tokens = prefix_tokens;
- } else {
- prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt
+ // need process the prompt
+ if (slot.state == IDLE && slot.command == LOAD_PROMPT)
+ {
+ slot.state = PROCESSING;
+ slot.command = NONE;
+ std::vector<llama_token> prompt_tokens;
+ slot.t_start_process_prompt = ggml_time_us();
+ slot.t_start_genereration = 0;
+
+ if (slot.infill)
+ {
+ bool suff_rm_leading_spc = true;
+ if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1)
+ {
+ params.input_suffix.erase(0, 1);
+ suff_rm_leading_spc = false;
}
+ auto prefix_tokens = tokenize(slot.params.input_prefix, false);
+ auto suffix_tokens = tokenize(slot.params.input_suffix, false);
- slot.n_past = 0;
- slot.n_prompt_tokens = prompt_tokens.size();
-
- LOG_VERBOSE("prompt tokenized", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
- {"n_ctx", slot.n_ctx},
- {"n_keep", slot.params.n_keep},
- {"n_prompt_tokens", slot.n_prompt_tokens},
- {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())},
- });
-
- // empty prompt passed -> release the slot and send empty response
- if (prompt_tokens.empty()) {
- LOG_INFO("empty prompt - releasing slot", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task}
- });
-
- slot.state = SLOT_STATE_PROCESSING;
- slot.command = SLOT_COMMAND_NONE;
- slot.release();
- slot.print_timings();
- send_final_response(slot);
- continue;
+ const int space_token = 29871; // TODO: this should not be hardcoded
+ if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {
+ suffix_tokens.erase(suffix_tokens.begin());
}
- if (slot.embedding) {
- // this prompt is too large to process - discard it
- if (slot.n_prompt_tokens > n_ubatch) {
- slot.state = SLOT_STATE_PROCESSING;
- slot.command = SLOT_COMMAND_NONE;
- slot.release();
- slot.print_timings();
- send_final_response(slot);
- continue;
- }
- } else {
- if (slot.params.n_keep < 0) {
- slot.params.n_keep = slot.n_prompt_tokens;
- }
- slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep);
-
- // if input prompt is too big, truncate it (if group attention self-extend is disabled)
- if (slot.ga_n == 1 && slot.n_prompt_tokens >= slot.n_ctx) {
- const int n_left = slot.n_ctx - slot.params.n_keep;
-
- const int n_block_size = n_left / 2;
- const int erased_blocks = (slot.n_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size;
-
- std::vector<llama_token> new_tokens(
- prompt_tokens.begin(),
- prompt_tokens.begin() + slot.params.n_keep);
-
- new_tokens.insert(
- new_tokens.end(),
- prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size,
- prompt_tokens.end());
-
- prompt_tokens = std::move(new_tokens);
+ prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));
+ prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS
+ prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));
+ prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());
+ prefix_tokens.push_back(llama_token_middle(model));
+ prompt_tokens = prefix_tokens;
+ }
+ else
+ {
+ prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt
+ }
- slot.truncated = true;
- slot.n_prompt_tokens = prompt_tokens.size();
+ slot.num_prompt_tokens = prompt_tokens.size();
- LOG_VERBOSE("input truncated", {
- {"id_slot", slot.id},
- {"id_task", slot.id_task},
- {"n_ctx", slot.n_ctx},
- {"n_keep", slot.params.n_keep},
- {"n_left", n_left},
- {"n_prompt_tokens", slot.n_prompt_tokens},
- {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())},
- });
+ if (slot.params.n_keep < 0)
+ {
+ slot.params.n_keep = slot.num_prompt_tokens;
+ }
+ slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep);
+
+ // if input prompt is too big, truncate it
+ if (slot.num_prompt_tokens >= slot.n_ctx)
+ {
+ const int n_left = slot.n_ctx - slot.params.n_keep;
+ const int n_block_size = n_left / 2;
+ const int erased_blocks = (slot.num_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size;
+
+ std::vector<llama_token> new_tokens(prompt_tokens.begin(), prompt_tokens.begin() + slot.params.n_keep);
+ new_tokens.insert(new_tokens.end(), prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size, prompt_tokens.end());
+
+ LOG_VERBOSE("input truncated", {
+ {"n_ctx", slot.n_ctx},
+ {"n_keep", slot.params.n_keep},
+ {"n_left", n_left},
+ {"new_tokens", tokens_to_str(ctx, new_tokens.cbegin(), new_tokens.cend())},
+ });
+ slot.truncated = true;
+ prompt_tokens = new_tokens;
- GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx);
- }
+ slot.num_prompt_tokens = prompt_tokens.size();
+ GGML_ASSERT(slot.num_prompt_tokens < slot.n_ctx);
+ }
- llama_sampling_reset(slot.ctx_sampling);
+ if (!slot.params.cache_prompt)
+ {
+ llama_sampling_reset(slot.ctx_sampling);
- if (!slot.params.cache_prompt) {
- slot.n_past_se = 0;
- slot.ga_i = 0;
- } else {
- GGML_ASSERT(slot.ga_n == 1);
+ slot.n_past = 0;
+ slot.n_past_se = 0;
+ slot.ga_i = 0;
+ slot.num_prompt_tokens_processed = slot.num_prompt_tokens;
+ }
+ else
+ {
+ // push the prompt into the sampling context (do not apply grammar)
+ for (auto &token : prompt_tokens)
+ {
+ llama_sampling_accept(slot.ctx_sampling, ctx, token, false);
+ }
- // reuse any previously computed tokens that are common with the new prompt
- slot.n_past = common_part(slot.cache_tokens, prompt_tokens);
+ slot.n_past = common_part(slot.cache_tokens, prompt_tokens);
- // push the prompt into the sampling context (do not apply grammar)
- for (int i = 0; i < slot.n_past; ++i) {
- llama_sampling_accept(slot.ctx_sampling, ctx, slot.cache_tokens[i], false);
- }
- }
+ // the last token of the cache is not in the KV cache until the next call to llama_decode
+ // (it was sampled, pushed into the "cache_tokens", but not yet put in the context)
+ if (slot.n_past > 0 && slot.n_past == (int32_t) slot.cache_tokens.size())
+ {
+ slot.n_past -= 1;
}
- if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0) {
- // we have to evaluate at least 1 token to generate logits.
- LOG_INFO("we have to evaluate at least 1 token to generate logits", {
- { "id_slot", slot.id },
- { "id_task", slot.id_task }
- });
-
- slot.n_past--;
- if (slot.ga_i > 0) {
- slot.n_past_se--;
+ slot.num_prompt_tokens_processed = slot.num_prompt_tokens - slot.n_past;
+
+ if (slot.ga_n != 1)
+ {
+ int ga_i = 0;
+ int32_t ga_n = slot.ga_n;
+ int32_t ga_w = slot.ga_w;
+ int32_t slot_npast = 0;
+ for (int k = 0; k < slot.n_past; ++k)
+ {
+ while (slot_npast >= ga_i + ga_w) {
+ const int bd = (ga_w/ga_n)*(ga_n - 1);
+ slot_npast -= bd;
+ ga_i += ga_w/ga_n;
+ }
+ slot_npast++;
}
+ slot.n_past_se = slot_npast;
+ slot.ga_i = ga_i;
}
- slot.n_prompt_tokens_processed = 0;
+ LOG_INFO("slot progression", {
+ { "slot_id", slot.id },
+ { "task_id", slot.task_id },
+ { "n_past", slot.n_past },
+ { "num_prompt_tokens_processed", slot.num_prompt_tokens_processed }
+ });
}
- if (slot.embedding) {
- // cannot fit the prompt in the current batch - will try next iter
- if (batch.n_tokens + slot.n_prompt_tokens > n_batch) {
- continue;
- }
- }
+ slot.cache_tokens = prompt_tokens;
- // keep only the common part
- int p0 = (int) system_tokens.size() + slot.n_past;
- if (!llama_kv_cache_seq_rm(ctx, slot.id + 1, p0, -1)) {
- // could not partially delete (likely using a non-Transformer model)
- llama_kv_cache_seq_rm(ctx, slot.id + 1, -1, -1);
-
- p0 = (int) system_tokens.size();
- if (p0 != 0) {
- // copy over the system prompt when there is one
- llama_kv_cache_seq_cp(ctx, 0, slot.id + 1, -1, -1);
+ if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0)
+ {
+ // we have to evaluate at least 1 token to generate logits.
+ LOG_INFO("we have to evaluate at least 1 token to generate logits", {
+ { "slot_id", slot.id },
+ { "task_id", slot.task_id }
+ });
+ slot.n_past--;
+ if (slot.ga_i > 0)
+ {
+ slot.n_past_se--;
}
-
- // there is no common part left (except for the system prompt)
- slot.n_past = 0;
- slot.n_past_se = 0;
- slot.ga_i = 0;
- // TODO: is the system prompt ever in the sampling context?
- llama_sampling_reset(slot.ctx_sampling);
}
- // remove the non-common part from the cache
- slot.cache_tokens.resize(slot.n_past);
-
+ int p0 = (int) system_tokens.size() + slot.n_past;
LOG_INFO("kv cache rm [p0, end)", {
- { "id_slot", slot.id },
- { "id_task", slot.id_task },
+ { "slot_id", slot.id },
+ { "task_id", slot.task_id },
{ "p0", p0 }
});
+ llama_kv_cache_seq_rm(ctx, slot.id, p0, -1);
+
+ LOG_VERBOSE("prompt ingested", {
+ {"n_past", slot.n_past},
+ {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},
+ {"to_eval", tokens_to_str(ctx, slot.cache_tokens.cbegin() + slot.n_past, slot.cache_tokens.cend())},
+ });
+
+ const bool has_images = process_images(slot);
+
+ // process the prefix of first image
+ std::vector<llama_token> prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens;
int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past;
@@ -1954,81 +1875,61 @@ struct server_context {
int32_t ga_n = slot.ga_n;
int32_t ga_w = slot.ga_w;
- // add prompt tokens for processing in the current batch
- // TODO: the self-extend stuff here is a mess - simplify and/or abstract it somehow
- for (; slot.n_past < slot.n_prompt_tokens && batch.n_tokens < n_batch; ++slot.n_past) {
- if (slot.ga_n != 1) {
+ for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past)
+ {
+ if (slot.ga_n != 1)
+ {
while (slot_npast >= ga_i + ga_w) {
const int bd = (ga_w/ga_n)*(ga_n - 1);
slot_npast -= bd;
ga_i += ga_w/ga_n;
}
}
-
- llama_batch_add(batch, prompt_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id + 1 }, false);
-
- if (slot.params.cache_prompt) {
- slot.cache_tokens.push_back(prompt_tokens[slot.n_past]);
- }
-
- slot.n_prompt_tokens_processed++;
+ llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, {slot.id }, false);
slot_npast++;
}
- LOG_VERBOSE("prompt processing progress", {
- {"id_slot", slot.id},
- {"n_past", slot.n_past},
- {"n_ctx", n_ctx},
- {"n_tokens", batch.n_tokens},
- {"progress", (float) slot.n_prompt_tokens_processed / slot.n_prompt_tokens},
- });
-
- // entire prompt has been processed - start decoding new tokens
- if (slot.n_past == slot.n_prompt_tokens) {
- slot.state = SLOT_STATE_PROCESSING;
- slot.command = SLOT_COMMAND_NONE;
-
- GGML_ASSERT(batch.n_tokens > 0);
+ if (has_images && !ingest_images(slot, n_batch))
+ {
+ LOG_ERROR("failed processing images", {
+ "slot_id", slot.id,
+ "task_id", slot.task_id,
+ });
+ // FIXME @phymbert: to be properly tested
+ // early returning without changing the slot state will block the slot for ever
+ // no one at the moment is checking the return value
+ return false;
+ }
- // extract the logits only for the last token
+ // extract the logits only for the last token
+ if (batch.n_tokens > 0)
+ {
batch.logits[batch.n_tokens - 1] = true;
-
- slot.n_decoded = 0;
- slot.i_batch = batch.n_tokens - 1;
-
- LOG_VERBOSE("prompt done", {
- {"id_slot", slot.id},
- {"n_past", slot.n_past},
- {"n_ctx", n_ctx},
- {"n_tokens", batch.n_tokens},
- });
}
- }
- if (batch.n_tokens >= n_batch) {
- break;
+ slot.n_decoded = 0;
+ slot.i_batch = batch.n_tokens - 1;
}
}
}
- if (batch.n_tokens == 0) {
- LOG_VERBOSE("no tokens to decode", {});
- return;
+ if (batch.n_tokens == 0)
+ {
+ all_slots_are_idle = true;
+ return true;
}
- LOG_VERBOSE("decoding batch", {
- {"n_tokens", batch.n_tokens},
- });
-
- // process the created batch of tokens
- for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) {
- const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);
+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)
+ {
+ const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));
- for (auto & slot : slots) {
- if (slot.ga_n != 1) {
+ for (auto & slot : slots)
+ {
+ if (slot.ga_n != 1)
+ {
// context extension via Self-Extend
- // TODO: simplify and/or abstract this
- while (slot.n_past_se >= slot.ga_i + slot.ga_w) {
+ while (slot.n_past_se >= slot.ga_i + slot.ga_w)
+ {
const int ib = (slot.ga_n * slot.ga_i) / slot.ga_w;
const int bd = (slot.ga_w / slot.ga_n) * (slot.ga_n - 1);
const int dd = (slot.ga_w / slot.ga_n) - ib * bd - slot.ga_w;
@@ -2038,9 +1939,9 @@ struct server_context {
LOG_TEE("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n, (slot.ga_i + ib * bd) / slot.ga_n, (slot.ga_i + ib * bd + slot.ga_w) / slot.ga_n);
LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd, slot.ga_i + ib * bd + slot.ga_w + dd, slot.n_past_se + ib * bd + dd);
- llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i, slot.n_past_se, ib * bd);
- llama_kv_cache_seq_div(ctx, slot.id + 1, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n);
- llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd);
+ llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd);
+ llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w,slot.ga_n);
+ llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w,slot.n_past_se + ib * bd, dd);
slot.n_past_se -= bd;
@@ -2048,12 +1949,12 @@ struct server_context {
LOG_TEE("\nn_past_old = %d, n_past = %d, ga_i = %d\n\n", slot.n_past_se + bd, slot.n_past_se, slot.ga_i);
}
-
slot.n_past_se += n_tokens;
}
}
- llama_batch batch_view = {
+ llama_batch batch_view =
+ {
n_tokens,
batch.token + i,
nullptr,
@@ -2066,17 +1967,13 @@ struct server_context {
const int ret = llama_decode(ctx, batch_view);
- if (ret != 0) {
- if (n_batch == 1 || ret < 0) {
+ if (ret != 0)
+ {
+ if (n_batch == 1 || ret < 0)
+ {
// if you get here, it means the KV cache is full - try increasing it via the context size
LOG_TEE("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret);
- for (auto & slot : slots) {
- slot.state = SLOT_STATE_PROCESSING;
- slot.command = SLOT_COMMAND_NONE;
- slot.release();
- send_error(slot, "Input prompt is too big compared to KV size. Please try increasing KV size.");
- }
- break; // break loop of n_batch
+ return false;
}
LOG_TEE("%s : failed to find free space in the KV cache, retrying with smaller n_batch = %d\n", __func__, n_batch / 2);
@@ -2084,21 +1981,23 @@ struct server_context {
// retry with half the batch size to try to find a free slot in the KV cache
n_batch /= 2;
i -= n_batch;
-
- continue; // continue loop of n_batch
+ continue;
}
- for (auto & slot : slots) {
- if (slot.state != SLOT_STATE_PROCESSING || slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) {
- continue; // continue loop of slots
+ for (auto & slot : slots)
+ {
+ if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens))
+ {
+ continue;
}
// prompt evaluated for embedding
- if (slot.embedding) {
- send_embedding(slot, batch_view);
+ if (slot.embedding)
+ {
+ send_embedding(slot);
slot.release();
slot.i_batch = -1;
- continue; // continue loop of slots
+ continue;
}
completion_token_output result;
@@ -2107,9 +2006,10 @@ struct server_context {
llama_sampling_accept(slot.ctx_sampling, ctx, id, true);
slot.n_decoded += 1;
- if (slot.n_decoded == 1) {
- slot.t_start_generation = ggml_time_us();
- slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3;
+ if (slot.n_decoded == 1)
+ {
+ slot.t_start_genereration = ggml_time_us();
+ slot.t_prompt_processing = (slot.t_start_genereration - slot.t_start_process_prompt) / 1e3;
metrics.on_prompt_eval(slot);
}
@@ -2117,19 +2017,19 @@ struct server_context {
result.tok = id;
const int32_t n_probs = slot.sparams.n_probs;
- if (slot.sparams.temp <= 0 && n_probs > 0) {
+ if (slot.sparams.temp <= 0 && n_probs > 0)
+ {
// for llama_sample_token_greedy we need to sort candidates
llama_sample_softmax(ctx, &cur_p);
}
- for (size_t i = 0; i < std::min(cur_p.size, (size_t) n_probs); ++i) {
- result.probs.push_back({
- cur_p.data[i].id,
- cur_p.data[i].p
- });
+ for (size_t i = 0; i < std::min(cur_p.size, (size_t)n_probs); ++i)
+ {
+ result.probs.push_back({cur_p.data[i].id, cur_p.data[i].p});
}
- if (!process_token(result, slot)) {
+ if (!process_token(result, slot))
+ {
slot.release();
slot.print_timings();
send_final_response(slot);
@@ -2140,22 +2040,18 @@ struct server_context {
}
}
- LOG_VERBOSE("run slots completed", {});
+ LOG_VERBOSE("slots updated", {});
+ return true;
}
- json model_meta() const {
- return json {
- {"vocab_type", llama_vocab_type (model)},
- {"n_vocab", llama_n_vocab (model)},
- {"n_ctx_train", llama_n_ctx_train (model)},
- {"n_embd", llama_n_embd (model)},
- {"n_params", llama_model_n_params(model)},
- {"size", llama_model_size (model)},
- };
+ void run_on_all_tasks_finished() {
+ update_slots();
}
};
-static void server_print_usage(const char * argv0, const gpt_params & params, const server_params & sparams) {
+static void server_print_usage(const char *argv0, const gpt_params &params,
+ const server_params &sparams)
+{
printf("usage: %s [options]\n", argv0);
printf("\n");
printf("options:\n");
@@ -2163,7 +2059,6 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co
printf(" -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled");
printf(" -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
printf(" -tb N, --threads-batch N number of threads to use during batch and prompt processing (default: same as --threads)\n");
- printf(" --threads-http N number of threads in the http server pool to process requests (default: max(hardware concurrency - 1, --parallel N + 2))\n");
printf(" -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx);
printf(" --rope-scaling {none,linear,yarn}\n");
printf(" RoPE frequency scaling method, defaults to linear unless specified by the model\n");
@@ -2173,17 +2068,15 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co
printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n");
printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow);
printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast);
- printf(" --pooling {none,mean,cls} pooling type for embeddings, use model default if unspecified\n");
- printf(" -dt N, --defrag-thold N\n");
- printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold);
- printf(" -b N, --batch-size N logical maximum batch size (default: %d)\n", params.n_batch);
- printf(" -ub N, --ubatch-size N physical maximum batch size (default: %d)\n", params.n_ubatch);
+ printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);
printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n");
printf(" not recommended: doubles context memory required and no measurable increase in quality\n");
- if (llama_supports_mlock()) {
+ if (llama_supports_mlock())
+ {
printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");
}
- if (llama_supports_mmap()) {
+ if (llama_supports_mmap())
+ {
printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");
}
printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n");
@@ -2205,23 +2098,17 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co
}
printf(" -m FNAME, --model FNAME\n");
printf(" model path (default: %s)\n", params.model.c_str());
- printf(" -mu MODEL_URL, --model-url MODEL_URL\n");
- printf(" model download url (default: %s)\n", params.model_url.c_str());
printf(" -a ALIAS, --alias ALIAS\n");
printf(" set an alias for the model, will be added as `model` field in completion response\n");
printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");
printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");
printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());
printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);
- printf(" --path PUBLIC_PATH path from which to serve static files (default: disabled)\n");
+ printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());
printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");
printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- printf(" --ssl-key-file FNAME path to file a PEM-encoded SSL private key\n");
- printf(" --ssl-cert-file FNAME path to file a PEM-encoded SSL certificate\n");
-#endif
printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);
- printf(" --embeddings enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");
+ printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");
printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);
printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");
printf(" -spf FNAME, --system-prompt-file FNAME\n");
@@ -2230,6 +2117,7 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co
printf(" KV cache data type for K (default: f16)\n");
printf(" -ctv TYPE, --cache-type-v TYPE\n");
printf(" KV cache data type for V (default: f16)\n");
+ printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");
printf(" --log-format log output format: json or text (default: json)\n");
printf(" --log-disable disables logging to a file.\n");
printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n");
@@ -2239,50 +2127,65 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co
printf(" --override-kv KEY=TYPE:VALUE\n");
printf(" advanced option to override model metadata by key. may be specified multiple times.\n");
printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");
- printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`\n");
- printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`\n");
+ printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`");
+ printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`");
printf(" --chat-template JINJA_TEMPLATE\n");
printf(" set custom jinja chat template (default: template taken from model's metadata)\n");
- printf(" only commonly used templates are accepted:\n");
- printf(" https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template\n");
+ printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n");
printf("\n");
}
-static void server_params_parse(int argc, char ** argv, server_params & sparams, gpt_params & params) {
- gpt_params default_params;
+static void server_params_parse(int argc, char **argv, server_params &sparams,
+ gpt_params &params, llama_server_context& llama)
+{
+ gpt_params default_params;
server_params default_sparams;
-
std::string arg;
bool invalid_param = false;
- for (int i = 1; i < argc; i++) {
+ for (int i = 1; i < argc; i++)
+ {
arg = argv[i];
- if (arg == "--port") {
- if (++i >= argc) {
+ if (arg == "--port")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
sparams.port = std::stoi(argv[i]);
- } else if (arg == "--host") {
- if (++i >= argc) {
+ }
+ else if (arg == "--host")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
sparams.hostname = argv[i];
- } else if (arg == "--path") {
- if (++i >= argc) {
+ }
+ else if (arg == "--path")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
sparams.public_path = argv[i];
- } else if (arg == "--api-key") {
- if (++i >= argc) {
+ }
+ else if (arg == "--api-key")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
- sparams.api_keys.push_back(argv[i]);
- } else if (arg == "--api-key-file") {
- if (++i >= argc) {
+ sparams.api_keys.emplace_back(argv[i]);
+ }
+ else if (arg == "--api-key-file")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
@@ -2299,59 +2202,53 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
}
}
key_file.close();
-
- }
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- else if (arg == "--ssl-key-file") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.ssl_key_file = argv[i];
- } else if (arg == "--ssl-cert-file") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.ssl_cert_file = argv[i];
}
-#endif
- else if (arg == "--timeout" || arg == "-to") {
- if (++i >= argc) {
+ else if (arg == "--timeout" || arg == "-to")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
sparams.read_timeout = std::stoi(argv[i]);
sparams.write_timeout = std::stoi(argv[i]);
- } else if (arg == "-m" || arg == "--model") {
- if (++i >= argc) {
+ }
+ else if (arg == "-m" || arg == "--model")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.model = argv[i];
- } else if (arg == "-mu" || arg == "--model-url") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.model_url = argv[i];
- } else if (arg == "-a" || arg == "--alias") {
- if (++i >= argc) {
+ }
+ else if (arg == "-a" || arg == "--alias")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.model_alias = argv[i];
- } else if (arg == "-h" || arg == "--help") {
+ }
+ else if (arg == "-h" || arg == "--help")
+ {
server_print_usage(argv[0], default_params, default_sparams);
exit(0);
- } else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size") {
- if (++i >= argc) {
+ }
+ else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.n_ctx = std::stoi(argv[i]);
- } else if (arg == "--rope-scaling") {
- if (++i >= argc) {
+ }
+ else if (arg == "--rope-scaling")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
@@ -2360,142 +2257,150 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }
else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }
else { invalid_param = true; break; }
- } else if (arg == "--rope-freq-base") {
- if (++i >= argc) {
+ }
+ else if (arg == "--rope-freq-base")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.rope_freq_base = std::stof(argv[i]);
- } else if (arg == "--rope-freq-scale") {
- if (++i >= argc) {
+ }
+ else if (arg == "--rope-freq-scale")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.rope_freq_scale = std::stof(argv[i]);
- } else if (arg == "--yarn-ext-factor") {
+ }
+ else if (arg == "--yarn-ext-factor")
+ {
if (++i >= argc) {
invalid_param = true;
break;
}
params.yarn_ext_factor = std::stof(argv[i]);
}
- else if (arg == "--yarn-attn-factor") {
+ else if (arg == "--yarn-attn-factor")
+ {
if (++i >= argc) {
invalid_param = true;
break;
}
params.yarn_attn_factor = std::stof(argv[i]);
- } else if (arg == "--yarn-beta-fast") {
+ }
+ else if (arg == "--yarn-beta-fast")
+ {
if (++i >= argc) {
invalid_param = true;
break;
}
params.yarn_beta_fast = std::stof(argv[i]);
- } else if (arg == "--yarn-beta-slow") {
+ }
+ else if (arg == "--yarn-beta-slow")
+ {
if (++i >= argc) {
invalid_param = true;
break;
}
params.yarn_beta_slow = std::stof(argv[i]);
- } else if (arg == "--pooling") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- std::string value(argv[i]);
- /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }
- else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }
- else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }
- else { invalid_param = true; break; }
- } else if (arg == "--defrag-thold" || arg == "-dt") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.defrag_thold = std::stof(argv[i]);
- } else if (arg == "--threads" || arg == "-t") {
+ }
+ else if (arg == "--threads" || arg == "-t")
+ {
if (++i >= argc)
{
invalid_param = true;
break;
}
params.n_threads = std::stoi(argv[i]);
- } else if (arg == "--grp-attn-n" || arg == "-gan") {
+ }
+ else if (arg == "--grp-attn-n" || arg == "-gan")
+ {
if (++i >= argc) {
invalid_param = true;
break;
}
params.grp_attn_n = std::stoi(argv[i]);
- } else if (arg == "--grp-attn-w" || arg == "-gaw") {
- if (++i >= argc) {
+ }
+ else if (arg == "--grp-attn-w" || arg == "-gaw")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.grp_attn_w = std::stoi(argv[i]);
- } else if (arg == "--threads-batch" || arg == "-tb") {
- if (++i >= argc) {
+ }
+ else if (arg == "--threads-batch" || arg == "-tb")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.n_threads_batch = std::stoi(argv[i]);
- } else if (arg == "--threads-http") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.n_threads_http = std::stoi(argv[i]);
- } else if (arg == "-b" || arg == "--batch-size") {
- if (++i >= argc) {
+ }
+ else if (arg == "-b" || arg == "--batch-size")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.n_batch = std::stoi(argv[i]);
- } else if (arg == "-ub" || arg == "--ubatch-size") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.n_ubatch = std::stoi(argv[i]);
- } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {
- if (++i >= argc) {
+ params.n_batch = std::min(512, params.n_batch);
+ }
+ else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
if (llama_supports_gpu_offload()) {
params.n_gpu_layers = std::stoi(argv[i]);
} else {
- LOG_WARNING(
- "Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "
- "See main README.md for information on enabling GPU BLAS support",
- {{"n_gpu_layers", params.n_gpu_layers}});
+ LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "
+ "See main README.md for information on enabling GPU BLAS support",
+ {{"n_gpu_layers", params.n_gpu_layers}});
}
- } else if (arg == "--split-mode" || arg == "-sm") {
+ }
+ else if (arg == "--split-mode" || arg == "-sm")
+ {
if (++i >= argc) {
invalid_param = true;
break;
}
std::string arg_next = argv[i];
- if (arg_next == "none") {
+ if (arg_next == "none")
+ {
params.split_mode = LLAMA_SPLIT_MODE_NONE;
- } else if (arg_next == "layer") {
+ }
+ else if (arg_next == "layer")
+ {
params.split_mode = LLAMA_SPLIT_MODE_LAYER;
- } else if (arg_next == "row") {
+ }
+ else if (arg_next == "row")
+ {
params.split_mode = LLAMA_SPLIT_MODE_ROW;
- } else {
+ }
+ else {
invalid_param = true;
break;
}
-#ifndef GGML_USE_CUBLAS
- fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting the split mode has no effect.\n");
-#endif // GGML_USE_CUBLAS
- } else if (arg == "--tensor-split" || arg == "-ts") {
- if (++i >= argc) {
+ }
+ else if (arg == "--tensor-split" || arg == "-ts")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
-#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL)
std::string arg_next = argv[i];
// split string by , and /
@@ -2504,67 +2409,91 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
std::vector<std::string> split_arg{it, {}};
GGML_ASSERT(split_arg.size() <= llama_max_devices());
- for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device) {
- if (i_device < split_arg.size()) {
+ for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device)
+ {
+ if (i_device < split_arg.size())
+ {
params.tensor_split[i_device] = std::stof(split_arg[i_device]);
- } else {
+ }
+ else
+ {
params.tensor_split[i_device] = 0.0f;
}
}
-#else
- LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a tensor split.\n", {});
-#endif // GGML_USE_CUBLAS
- } else if (arg == "--main-gpu" || arg == "-mg") {
- if (++i >= argc) {
+ }
+ else if (arg == "--main-gpu" || arg == "-mg")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
-#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL)
params.main_gpu = std::stoi(argv[i]);
-#else
- LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a main GPU.", {});
-#endif
- } else if (arg == "--lora") {
- if (++i >= argc) {
+ }
+ else if (arg == "--lora")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.lora_adapter.emplace_back(argv[i], 1.0f);
params.use_mmap = false;
- } else if (arg == "--lora-scaled") {
- if (++i >= argc) {
+ }
+ else if (arg == "--lora-scaled")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
const char * lora_adapter = argv[i];
- if (++i >= argc) {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));
params.use_mmap = false;
- } else if (arg == "--lora-base") {
- if (++i >= argc) {
+ }
+ else if (arg == "--lora-base")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.lora_base = argv[i];
- } else if (arg == "-v" || arg == "--verbose") {
+ }
+ else if (arg == "-v" || arg == "--verbose")
+ {
#if SERVER_VERBOSE != 1
LOG_WARNING("server.cpp is not built with verbose logging.", {});
#else
server_verbose = true;
#endif
- } else if (arg == "--nobrowser") {
+ }
+ else if (arg == "--nobrowser")
+ {
sparams.nobrowser = true;
- } else if (arg == "--unsecure") {
+ }
+ else if (arg == "--unsecure")
+ {
FLAG_unsecure = true;
- } else if (arg == "--server") {
- } else if (arg == "--nocompile") {
+ }
+ else if (arg == "--server")
+ {
+ }
+ else if (arg == "--nocompile")
+ {
FLAG_nocompile = true;
- } else if (arg == "--recompile") {
+ }
+ else if (arg == "--recompile")
+ {
FLAG_recompile = true;
- } else if (arg == "--gpu") {
+ }
+ else if (arg == "--gpu")
+ {
if (++i >= argc)
{
invalid_param = true;
@@ -2576,39 +2505,55 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);
exit(1);
}
- } else if (arg == "--mlock") {
+ }
+ else if (arg == "--mlock")
+ {
params.use_mlock = true;
- } else if (arg == "--no-mmap") {
+ }
+ else if (arg == "--no-mmap")
+ {
params.use_mmap = false;
- } else if (arg == "--numa") {
+ }
+ else if (arg == "--numa") {
if (++i >= argc) {
invalid_param = true;
break;
} else {
std::string value(argv[i]);
/**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }
- else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }
- else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }
+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }
+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }
else { invalid_param = true; break; }
}
- } else if (arg == "--embedding" || arg == "--embeddings") {
+ }
+ else if (arg == "--embedding")
+ {
params.embedding = true;
- } else if (arg == "-cb" || arg == "--cont-batching") {
+ }
+ else if (arg == "-cb" || arg == "--cont-batching")
+ {
params.cont_batching = true;
- } else if (arg == "-np" || arg == "--parallel") {
- if (++i >= argc) {
+ }
+ else if (arg == "-np" || arg == "--parallel")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.n_parallel = std::stoi(argv[i]);
- } else if (arg == "-n" || arg == "--n-predict") {
- if (++i >= argc) {
+ } else if (arg == "-n" || arg == "--n-predict")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
params.n_predict = std::stoi(argv[i]);
- } else if (arg == "-spf" || arg == "--system-prompt-file") {
- if (++i >= argc) {
+ } else if (arg == "-spf" || arg == "--system-prompt-file")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
@@ -2618,39 +2563,67 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
invalid_param = true;
break;
}
- std::string system_prompt;
+ std::string systm_content;
std::copy(
std::istreambuf_iterator<char>(file),
std::istreambuf_iterator<char>(),
- std::back_inserter(system_prompt)
+ std::back_inserter(systm_content)
);
- sparams.system_prompt = system_prompt;
- } else if (arg == "-ctk" || arg == "--cache-type-k") {
+ llama.process_system_prompt_data(json::parse(systm_content));
+ }
+ else if (arg == "-ctk" || arg == "--cache-type-k") {
params.cache_type_k = argv[++i];
- } else if (arg == "-ctv" || arg == "--cache-type-v") {
+ }
+ else if (arg == "-ctv" || arg == "--cache-type-v") {
params.cache_type_v = argv[++i];
- } else if (arg == "--log-format") {
- if (++i >= argc) {
+ }
+ else if(arg == "--mmproj")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ params.mmproj = argv[i];
+ }
+ else if (arg == "--log-format")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
- if (std::strcmp(argv[i], "json") == 0) {
+ if (std::strcmp(argv[i], "json") == 0)
+ {
server_log_json = true;
- } else if (std::strcmp(argv[i], "text") == 0) {
+ }
+ else if (std::strcmp(argv[i], "text") == 0)
+ {
server_log_json = false;
- } else {
+ }
+ else
+ {
invalid_param = true;
break;
}
- } else if (arg == "--log-disable") {
+ }
+ else if (arg == "--log-disable")
+ {
log_set_target(stdout);
LOG_INFO("logging to file is disabled.", {});
- } else if (arg == "--slots-endpoint-disable") {
+ }
+ else if (arg == "--slots-endpoint-disable")
+ {
sparams.slots_endpoint = false;
- } else if (arg == "--metrics") {
+ }
+ else if (arg == "--metrics")
+ {
sparams.metrics_endpoint = true;
- } else if (arg == "--chat-template") {
- if (++i >= argc) {
+ }
+ else if (arg == "--chat-template")
+ {
+ if (++i >= argc)
+ {
invalid_param = true;
break;
}
@@ -2661,7 +2634,9 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
break;
}
sparams.chat_template = argv[i];
- } else if (arg == "--override-kv") {
+ }
+ else if (arg == "--override-kv")
+ {
if (++i >= argc) {
invalid_param = true;
break;
@@ -2672,7 +2647,6 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
invalid_param = true;
break;
}
-
struct llama_model_kv_override kvo;
std::strncpy(kvo.key, argv[i], sep - argv[i]);
kvo.key[sep - argv[i]] = 0;
@@ -2703,28 +2677,71 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
break;
}
params.kv_overrides.push_back(kvo);
- } else {
+ }
+ else
+ {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
server_print_usage(argv[0], default_params, default_sparams);
exit(1);
}
}
+ params.embedding = true; // [jart] #243 always enable embedding mode
+ params.n_gpu_layers = llamafile_gpu_layers(params.n_gpu_layers);
+
if (!params.kv_overrides.empty()) {
params.kv_overrides.emplace_back();
params.kv_overrides.back().key[0] = 0;
}
- if (invalid_param) {
+ if (invalid_param)
+ {
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
server_print_usage(argv[0], default_params, default_sparams);
exit(1);
}
}
-static void log_server_request(const httplib::Request & req, const httplib::Response & res) {
+/* llama.cpp completion api semantics */
+static json format_partial_response(
+ llama_server_context &llama, llama_client_slot *slot, const std::string &content, const std::vector<completion_token_output> &probs
+) {
+ json res = json
+ {
+ {"content", content },
+ {"stop", false},
+ {"slot_id", slot->id },
+ {"multimodal", llama.multimodal }
+ };
+
+ if (slot->sparams.n_probs > 0)
+ {
+ res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);
+ }
+
+ return res;
+}
+
+static json format_tokenizer_response(const std::vector<llama_token> &tokens)
+{
+ return json {
+ {"tokens", tokens}
+ };
+}
+
+static json format_detokenized_response(std::string content)
+{
+ return json {
+ {"content", content}
+ };
+}
+
+
+static void log_server_request(const httplib::Request &req, const httplib::Response &res)
+{
// skip GH copilot requests when using default port
- if (req.path == "/v1/health" || req.path == "/v1/completions") {
+ if (req.path == "/v1/health" || req.path == "/v1/completions")
+ {
return;
}
@@ -2743,164 +2760,308 @@ static void log_server_request(const httplib::Request & req, const httplib::Resp
});
}
-std::function<void(int)> shutdown_handler;
-std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
+struct token_translator
+{
+ llama_context * ctx;
+ std::string operator()(llama_token tok) const { return llama_token_to_piece(ctx, tok); }
+ std::string operator()(const completion_token_output &cto) const { return (*this)(cto.tok); }
+};
-inline void signal_handler(int signal) {
- if (is_terminating.test_and_set()) {
- // in case it hangs, we can force terminate the server by hitting Ctrl+C twice
- // this is for better developer experience, we can remove when the server is stable enough
- fprintf(stderr, "Received second interrupt, terminating immediately.\n");
- exit(1);
+static void append_to_generated_text_from_generated_token_probs(llama_server_context &llama, llama_client_slot *slot)
+{
+ auto & gtps = slot->generated_token_probs;
+ auto translator = token_translator{llama.ctx};
+ auto add_strlen = [=](size_t sum, const completion_token_output & cto) { return sum + translator(cto).size(); };
+ const size_t len = std::accumulate(gtps.begin(), gtps.end(), size_t(0), add_strlen);
+ if (slot->generated_text.capacity() < slot->generated_text.size() + len)
+ {
+ slot->generated_text.reserve(slot->generated_text.size() + len);
+ }
+ for (const completion_token_output & cto : gtps)
+ {
+ slot->generated_text += translator(cto);
}
-
- shutdown_handler(signal);
}
+std::function<void(int)> shutdown_handler;
+inline void signal_handler(int signal) { shutdown_handler(signal); }
+
static const char *sockaddr2str(const struct sockaddr *sa, char *buf, size_t size) {
- if (sa->sa_family == AF_INET) {
- return inet_ntop(AF_INET, &(((const struct sockaddr_in *)sa)->sin_addr),
- buf, size);
- } else if (sa->sa_family == AF_INET6) {
- return inet_ntop(AF_INET6, &(((const struct sockaddr_in6 *)sa)->sin6_addr),
- buf, size);
- } else {
- return 0;
- }
+ if (sa->sa_family == AF_INET) {
+ return inet_ntop(AF_INET, &(((const struct sockaddr_in *)sa)->sin_addr),
+ buf, size);
+ } else if (sa->sa_family == AF_INET6) {
+ return inet_ntop(AF_INET6, &(((const struct sockaddr_in6 *)sa)->sin6_addr),
+ buf, size);
+ } else {
+ return 0;
+ }
}
-int server_cli(int argc, char ** argv) {
+int server_cli(int argc, char **argv)
+{
#if SERVER_VERBOSE != 1
log_disable();
#endif
// own arguments required by this example
- gpt_params params;
+ gpt_params params;
server_params sparams;
// struct that contains llama context and inference
- server_context ctx_server;
+ llama_server_context llama;
- server_params_parse(argc, argv, sparams, params);
+ server_params_parse(argc, argv, sparams, params, llama);
- if (!sparams.system_prompt.empty()) {
- ctx_server.system_prompt_set(json::parse(sparams.system_prompt));
- }
-
- if (params.model_alias == "unknown") {
+ if (params.model_alias == "unknown")
+ {
params.model_alias = params.model;
}
llama_backend_init();
llama_numa_init(params.numa);
- LOG_INFO("build info", {
- {"build", LLAMA_BUILD_NUMBER},
- {"commit", LLAMA_COMMIT}
- });
+ LOG_INFO("build info", {{"build", LLAMA_BUILD_NUMBER},
+ {"commit", LLAMA_COMMIT}});
LOG_INFO("system info", {
- {"n_threads", params.n_threads},
- {"n_threads_batch", params.n_threads_batch},
- {"total_threads", std::thread::hardware_concurrency()},
- {"system_info", llama_print_system_info()},
- });
+ {"n_threads", params.n_threads},
+ {"n_threads_batch", params.n_threads_batch},
+ {"total_threads", std::thread::hardware_concurrency()},
+ {"system_info", llama_print_system_info()},
+ });
- std::unique_ptr<httplib::Server> svr;
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- if (sparams.ssl_key_file != "" && sparams.ssl_cert_file != "") {
- LOG_INFO("Running with SSL", {{"key", sparams.ssl_key_file}, {"cert", sparams.ssl_cert_file}});
- svr.reset(
- new httplib::SSLServer(sparams.ssl_cert_file.c_str(), sparams.ssl_key_file.c_str())
- );
- } else {
- LOG_INFO("Running without SSL", {});
- svr.reset(new httplib::Server());
- }
-#else
- svr.reset(new httplib::Server());
-#endif
+ httplib::Server svr;
std::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};
- svr->set_default_headers({{"Server", "llama.cpp"}});
+ svr.set_default_headers({{"Server", "llama.cpp"}});
// CORS preflight
- svr->Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ svr.Options(R"(.*)", [](const httplib::Request &req, httplib::Response &res) {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
res.set_header("Access-Control-Allow-Credentials", "true");
- res.set_header("Access-Control-Allow-Methods", "POST");
- res.set_header("Access-Control-Allow-Headers", "*");
- return res.set_content("", "application/json; charset=utf-8");
+ res.set_header("Access-Control-Allow-Methods", "POST");
+ res.set_header("Access-Control-Allow-Headers", "*");
});
- svr->set_logger(log_server_request);
+ svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) {
+ server_state current_state = state.load();
+ switch(current_state) {
+ case SERVER_STATE_READY: {
+ // request slots data using task queue
+ task_server task;
+ task.id = llama.queue_tasks.get_new_id();
+ task.type = TASK_TYPE_METRICS;
+ task.target_id = -1;
+
+ llama.queue_results.add_waiting_task_id(task.id);
+ llama.queue_tasks.post(task);
- auto res_error = [](httplib::Response & res, json error_data) {
- json final_response {{"error", error_data}};
- res.set_content(final_response.dump(), "application/json; charset=utf-8");
- res.status = json_value(error_data, "code", 500);
- };
+ // get the result
+ task_result result = llama.queue_results.recv(task.id);
+ llama.queue_results.remove_waiting_task_id(task.id);
- svr->set_exception_handler([&res_error](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) {
- std::string message;
- try {
- std::rethrow_exception(std::move(ep));
- } catch (std::exception & e) {
- message = e.what();
- } catch (...) {
- message = "Unknown Exception";
- }
+ int n_idle_slots = result.result_json["idle"];
+ int n_processing_slots = result.result_json["processing"];
- json formatted_error = format_error_response(message, ERROR_TYPE_SERVER);
- LOG_VERBOSE("Got exception", formatted_error);
- res_error(res, formatted_error);
- });
+ json health = {
+ {"status", "ok"},
+ {"slots_idle", n_idle_slots},
+ {"slots_processing", n_processing_slots}};
+ res.status = 200; // HTTP OK
+ if (sparams.slots_endpoint && req.has_param("include_slots")) {
+ health["slots"] = result.result_json["slots"];
+ }
- svr->set_error_handler([&res_error](const httplib::Request &, httplib::Response & res) {
- if (res.status == 404) {
- res_error(res, format_error_response("File Not Found", ERROR_TYPE_NOT_FOUND));
+ if (n_idle_slots == 0) {
+ health["status"] = "no slot available";
+ if (req.has_param("fail_on_no_slot")) {
+ res.status = 503; // HTTP Service Unavailable
+ }
+ }
+ res.set_content(health.dump(), "application/json");
+ break;
+ }
+ case SERVER_STATE_LOADING_MODEL:
+ res.set_content(R"({"status": "loading model"})", "application/json");
+ res.status = 503; // HTTP Service Unavailable
+ break;
+ case SERVER_STATE_ERROR:
+ res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json");
+ res.status = 500; // HTTP Internal Server Error
+ break;
}
- // for other error codes, we skip processing here because it's already done by res_error()
});
- // set timeouts and change hostname and port
- svr->set_read_timeout (sparams.read_timeout);
- svr->set_write_timeout(sparams.write_timeout);
+ if (sparams.slots_endpoint) {
+ svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) {
+ // request slots data using task queue
+ task_server task;
+ task.id = llama.queue_tasks.get_new_id();
+ task.type = TASK_TYPE_METRICS;
+ task.target_id = -1;
- std::unordered_map<std::string, std::string> log_data;
+ llama.queue_results.add_waiting_task_id(task.id);
+ llama.queue_tasks.post(task);
- log_data["hostname"] = sparams.hostname;
- log_data["port"] = std::to_string(sparams.port);
+ // get the result
+ task_result result = llama.queue_results.recv(task.id);
+ llama.queue_results.remove_waiting_task_id(task.id);
- if (sparams.api_keys.size() == 1) {
- auto key = sparams.api_keys[0];
- log_data["api_key"] = "api_key: ****" + key.substr(std::max((int)(key.length() - 4), 0));
- } else if (sparams.api_keys.size() > 1) {
- log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";
+ res.set_content(result.result_json["slots"].dump(), "application/json");
+ res.status = 200; // HTTP OK
+ });
}
- // load the model
- if (!ctx_server.load_model(params)) {
- state.store(SERVER_STATE_ERROR);
- return 1;
- } else {
- ctx_server.init();
- state.store(SERVER_STATE_READY);
- }
+ if (sparams.metrics_endpoint) {
+ svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) {
+ // request slots data using task queue
+ task_server task;
+ task.id = llama.queue_tasks.get_new_id();
+ task.type = TASK_TYPE_METRICS;
+ task.target_id = -1;
- LOG_INFO("model loaded", {});
+ llama.queue_results.add_waiting_task_id(task.id);
+ llama.queue_tasks.post(task);
- for (int i = 0;; ++i) {
- if (svr->bind_to_port(sparams.hostname, sparams.port)) {
- break;
- } else if (i < 10) {
- ++sparams.port;
- } else {
- fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);
- return 1;
+ // get the result
+ task_result result = llama.queue_results.recv(task.id);
+ llama.queue_results.remove_waiting_task_id(task.id);
+
+ json data = result.result_json;
+
+ uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"];
+ uint64_t t_prompt_processing = data["t_prompt_processing"];
+
+ uint64_t n_tokens_predicted = data["n_tokens_predicted"];
+ uint64_t t_tokens_generation = data["t_tokens_generation"];
+
+ int32_t kv_cache_used_cells = data["kv_cache_used_cells"];
+
+ // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
+ json all_metrics_def = json {
+ {"counter", {{
+ {"name", "prompt_tokens_total"},
+ {"help", "Number of prompt tokens processed."},
+ {"value", data["n_prompt_tokens_processed_total"]}
+ }, {
+ {"name", "tokens_predicted_total"},
+ {"help", "Number of generation tokens processed."},
+ {"value", data["n_tokens_predicted_total"]}
+ }}},
+ {"gauge", {{
+ {"name", "prompt_tokens_seconds"},
+ {"help", "Average prompt throughput in tokens/s."},
+ {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0}
+ },{
+ {"name", "predicted_tokens_seconds"},
+ {"help", "Average generation throughput in tokens/s."},
+ {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0}
+ },{
+ {"name", "kv_cache_usage_ratio"},
+ {"help", "KV-cache usage. 1 means 100 percent usage."},
+ {"value", 1. * kv_cache_used_cells / params.n_ctx}
+ },{
+ {"name", "kv_cache_tokens"},
+ {"help", "KV-cache tokens."},
+ {"value", data["kv_cache_tokens_count"]}
+ },{
+ {"name", "requests_processing"},
+ {"help", "Number of request processing."},
+ {"value", data["processing"]}
+ },{
+ {"name", "requests_deferred"},
+ {"help", "Number of request deferred."},
+ {"value", data["deferred"]}
+ }}}
+ };
+
+ std::stringstream prometheus;
+ for (const auto& el : all_metrics_def.items()) {
+ const auto& type = el.key();
+ const auto& metrics_def = el.value();
+ for (const auto& metric_def : metrics_def) {
+ std::string name = metric_def["name"];
+ std::string help = metric_def["help"];
+ prometheus << "# HELP llamacpp:" << name << " " << help << "\n"
+ << "# TYPE llamacpp:" << name << " " << type << "\n"
+ << "llamacpp:" << name << " " << metric_def["value"] << "\n";
+ }
+ }
+
+ res.set_content(prometheus.str(), "text/plain; version=0.0.4");
+ res.status = 200; // HTTP OK
+ });
+ }
+
+ svr.set_logger(log_server_request);
+
+ svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep)
+ {
+ const char fmt[] = "500 Internal Server Error\n%s";
+ char buf[BUFSIZ];
+ try
+ {
+ std::rethrow_exception(std::move(ep));
+ }
+ catch (std::exception &e)
+ {
+ snprintf(buf, sizeof(buf), fmt, e.what());
+ }
+ catch (...)
+ {
+ snprintf(buf, sizeof(buf), fmt, "Unknown Exception");
+ }
+ res.set_content(buf, "text/plain; charset=utf-8");
+ res.status = 500;
+ });
+
+ svr.set_error_handler([](const httplib::Request &, httplib::Response &res)
+ {
+ if (res.status == 401)
+ {
+ res.set_content("Unauthorized", "text/plain; charset=utf-8");
+ }
+ if (res.status == 400)
+ {
+ res.set_content("Invalid request", "text/plain; charset=utf-8");
+ }
+ else if (res.status == 404)
+ {
+ res.set_content("File Not Found", "text/plain; charset=utf-8");
+ res.status = 404;
+ }
+ });
+
+ // load the model
+ if (!llama.load_model(params))
+ {
+ state.store(SERVER_STATE_ERROR);
+ return 1;
+ } else {
+ llama.initialize();
+ state.store(SERVER_STATE_READY);
+ LOG_INFO("model loaded", {});
+ }
+
+ // set timeouts and change hostname and port
+ svr.set_read_timeout (sparams.read_timeout);
+ svr.set_write_timeout(sparams.write_timeout);
+
+ for (int i = 0;; ++i) {
+ if (svr.bind_to_port(sparams.hostname, sparams.port)) {
+ break;
+ } else if (i < 10) {
+ ++sparams.port;
+ } else {
+ fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);
+ return 1;
}
}
+ // Set the base directory for serving static files
+ svr.set_base_dir(sparams.public_path);
+
// to make it ctrl+clickable:
const char *connect_host;
if (sparams.hostname != "0.0.0.0") {
@@ -2929,66 +3090,107 @@ int server_cli(int argc, char ** argv) {
}
}
- const auto model_meta = ctx_server.model_meta();
+ std::unordered_map<std::string, std::string> log_data;
+ log_data["hostname"] = sparams.hostname;
+ log_data["port"] = std::to_string(sparams.port);
- // if a custom chat template is not supplied, we will use the one that comes with the model (if any)
- if (sparams.chat_template.empty()) {
- if (!ctx_server.validate_model_chat_template()) {
- LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {});
- sparams.chat_template = "chatml";
+ if (sparams.api_keys.size() == 1) {
+ log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4);
+ } else if (sparams.api_keys.size() > 1) {
+ log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";
+ }
+
+ if (!FLAG_unsecure) {
+ if (IsXnu()) {
+ // Cosmopolitan libc explicitly does not support cosmo_dlopen on x64
+ // macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt
+ // to enable sandboxing and ignore the failure on x64 macOS.
+ // (Alternatively, the sandbox-exec(1) command could be used to
+ // launch llamafile with a sandbox profile.) A goal of llamafile is
+ // to work easily out-of-the-box and it already operates without a
+ // sandbox on macOS so don't require --unsecure on x64 macOS to
+ // workaround the failure. If support is added later, it will be
+ // used without further changes. On Apple Silicon, if sandbox
+ // initialization fails, exit and recommend using --unsecure to
+ // disable sandboxing.
+ std::string error_msg;
+ int rv = mac_sandbox_init(error_msg);
+ if (rv != 0) {
+ fprintf(stderr, "Sandbox initialization failed:\n"
+ "rv: %d, error: %s.\n"
+ "Disable sandboxing with the --unsecure option.\n",
+ rv, error_msg.c_str());
+ if (IsXnuSilicon()) {
+ exit(1);
+ } else {
+ LOG_WARNING("Sandbox initialization failed. Not supported "
+ "on x64 macOS. Continuing.", {});
+ }
+ }
+ } else if (!llamafile_has_gpu()) {
+ printf("In the sandboxing block!\n");
+ // Enables pledge() security on Linux and OpenBSD.
+ // - We do this *after* binding the server socket.
+ // - We do this *after* opening the log file for writing.
+ // - We do this *after* opening a tab in the user browser.
+ // - We do this *before* loading any weights or graphdefs.
+ // In effect, what this does is:
+ // - Filesystem access is disabled entirely (except ZipOS).
+ // - On Linux, network access is restricted to accept() only.
+ // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.
+ char promises[32];
+ if (IsOpenbsd()) {
+ strlcpy(promises, "stdio inet", sizeof(promises));
+ } else {
+ strlcpy(promises, "stdio anet", sizeof(promises));
+ }
+ if (!startswith(sparams.public_path.c_str(), "/zip/")) {
+ strlcat(promises, " rpath", sizeof(promises));
+ }
+ __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;
+ if (pledge(0, 0)) {
+ LOG_TEE("warning: this OS doesn't support pledge() security\n");
+ } else if (pledge(promises, 0)) {
+ perror("pledge");
+ exit(1);
+ }
}
}
- // print sample chat example to make it clear which template is used
- {
- json chat;
- chat.push_back({{"role", "system"}, {"content", "You are a helpful assistant"}});
- chat.push_back({{"role", "user"}, {"content", "Hello"}});
- chat.push_back({{"role", "assistant"}, {"content", "Hi there"}});
- chat.push_back({{"role", "user"}, {"content", "How are you?"}});
+ LOG_INFO("HTTP server listening", log_data);
+ // run the HTTP server in a thread - see comment below
+ std::thread t([&]()
+ {
+ if (!svr.listen_after_bind())
+ {
+ state.store(SERVER_STATE_ERROR);
+ return 1;
+ }
- const std::string chat_example = format_chat(ctx_server.model, sparams.chat_template, chat);
+ return 0;
+ });
- LOG_INFO("chat template", {
- {"chat_example", chat_example},
- {"built_in", sparams.chat_template.empty()},
- });
+ // launch browser tab
+ if (!sparams.nobrowser) {
+ char url[128];
+ snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);
+ llamafile_launch_browser(url);
}
- //
- // Middlewares
- //
-
- auto middleware_validate_api_key = [&sparams, &res_error](const httplib::Request & req, httplib::Response & res) {
- // TODO: should we apply API key to all endpoints, including "/health" and "/models"?
- static const std::set<std::string> protected_endpoints = {
- "/props",
- "/completion",
- "/completions",
- "/v1/completions",
- "/chat/completions",
- "/v1/chat/completions",
- "/infill",
- "/tokenize",
- "/detokenize",
- "/embedding",
- "/embeddings",
- "/v1/embeddings",
- };
+ if (sparams.chat_template.empty()) { // custom chat template is not supplied
+ // check if the template comes with the model is supported by us
+ llama.validate_model_chat_template(sparams);
+ }
+ // Middleware for API key validation
+ auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool {
// If API key is not set, skip validation
if (sparams.api_keys.empty()) {
return true;
}
- // If path is not in protected_endpoints list, skip validation
- if (protected_endpoints.find(req.path) == protected_endpoints.end()) {
- return true;
- }
-
// Check for API key in the header
auto auth_header = req.get_header_value("Authorization");
-
std::string prefix = "Bearer ";
if (auth_header.substr(0, prefix.size()) == prefix) {
std::string received_api_key = auth_header.substr(prefix.size());
@@ -2998,663 +3200,427 @@ int server_cli(int argc, char ** argv) {
}
// API key is invalid or not provided
- // TODO: make another middleware for CORS related logic
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
- res_error(res, format_error_response("Invalid API Key", ERROR_TYPE_AUTHENTICATION));
+ res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8");
+ res.status = 401; // Unauthorized
LOG_WARNING("Unauthorized: Invalid API Key", {});
return false;
};
- // register server middlewares
- svr->set_pre_routing_handler([&middleware_validate_api_key](const httplib::Request & req, httplib::Response & res) {
- if (!middleware_validate_api_key(req, res)) {
- return httplib::Server::HandlerResponse::Handled;
- }
- return httplib::Server::HandlerResponse::Unhandled;
- });
-
- //
- // Route handlers (or controllers)
- //
-
- const auto handle_health = [&](const httplib::Request & req, httplib::Response & res) {
- server_state current_state = state.load();
- switch (current_state) {
- case SERVER_STATE_READY:
- {
- // request slots data using task queue
- server_task task;
- task.id = ctx_server.queue_tasks.get_new_id();
- task.type = SERVER_TASK_TYPE_METRICS;
- task.id_target = -1;
-
- ctx_server.queue_results.add_waiting_task_id(task.id);
- ctx_server.queue_tasks.post(task);
-
- // get the result
- server_task_result result = ctx_server.queue_results.recv(task.id);
- ctx_server.queue_results.remove_waiting_task_id(task.id);
-
- const int n_idle_slots = result.data["idle"];
- const int n_processing_slots = result.data["processing"];
-
- json health = {
- {"status", "ok"},
- {"slots_idle", n_idle_slots},
- {"slots_processing", n_processing_slots}
- };
+ svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ json data = {
+ { "user_name", llama.name_user.c_str() },
+ { "assistant_name", llama.name_assistant.c_str() },
+ { "default_generation_settings", llama.default_generation_settings_for_props },
+ { "total_slots", llama.params.n_parallel }
+ };
+ res.set_content(data.dump(), "application/json; charset=utf-8");
+ });
- res.status = 200; // HTTP OK
- if (sparams.slots_endpoint && req.has_param("include_slots")) {
- health["slots"] = result.data["slots"];
+ svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ if (!validate_api_key(req, res)) {
+ return;
+ }
+ json data = json::parse(req.body);
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, data, false, false, -1);
+ if (!json_value(data, "stream", false)) {
+ std::string completion_text;
+ task_result result = llama.queue_results.recv(task_id);
+ if (!result.error && result.stop) {
+ res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");
}
-
- if (n_idle_slots == 0) {
- health["status"] = "no slot available";
- if (req.has_param("fail_on_no_slot")) {
- res.status = 503; // HTTP Service Unavailable
- }
+ else
+ {
+ res.status = 404;
+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");
}
-
- res.set_content(health.dump(), "application/json");
- break;
- }
- case SERVER_STATE_LOADING_MODEL:
- {
- res_error(res, format_error_response("Loading model", ERROR_TYPE_UNAVAILABLE));
- } break;
- case SERVER_STATE_ERROR:
- {
- res_error(res, format_error_response("Model failed to load", ERROR_TYPE_SERVER));
- } break;
- }
- };
-
- const auto handle_slots = [&](const httplib::Request &, httplib::Response & res) {
- if (!sparams.slots_endpoint) {
- res_error(res, format_error_response("This server does not support slots endpoint.", ERROR_TYPE_NOT_SUPPORTED));
- return;
- }
-
- // request slots data using task queue
- server_task task;
- task.id = ctx_server.queue_tasks.get_new_id();
- task.id_multi = -1;
- task.id_target = -1;
- task.type = SERVER_TASK_TYPE_METRICS;
-
- ctx_server.queue_results.add_waiting_task_id(task.id);
- ctx_server.queue_tasks.post(task);
-
- // get the result
- server_task_result result = ctx_server.queue_results.recv(task.id);
- ctx_server.queue_results.remove_waiting_task_id(task.id);
-
- res.set_content(result.data["slots"].dump(), "application/json");
- res.status = 200; // HTTP OK
- };
-
- const auto handle_metrics = [&](const httplib::Request &, httplib::Response & res) {
- if (!sparams.metrics_endpoint) {
- res_error(res, format_error_response("This server does not support metrics endpoint.", ERROR_TYPE_NOT_SUPPORTED));
- return;
- }
-
- // request slots data using task queue
- server_task task;
- task.id = ctx_server.queue_tasks.get_new_id();
- task.id_multi = -1;
- task.id_target = -1;
- task.type = SERVER_TASK_TYPE_METRICS;
- task.data.push_back({{"reset_bucket", true}});
-
- ctx_server.queue_results.add_waiting_task_id(task.id);
- ctx_server.queue_tasks.post(task);
-
- // get the result
- server_task_result result = ctx_server.queue_results.recv(task.id);
- ctx_server.queue_results.remove_waiting_task_id(task.id);
-
- json data = result.data;
-
- const uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"];
- const uint64_t t_prompt_processing = data["t_prompt_processing"];
-
- const uint64_t n_tokens_predicted = data["n_tokens_predicted"];
- const uint64_t t_tokens_generation = data["t_tokens_generation"];
-
- const int32_t kv_cache_used_cells = data["kv_cache_used_cells"];
-
- // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
- json all_metrics_def = json {
- {"counter", {{
- {"name", "prompt_tokens_total"},
- {"help", "Number of prompt tokens processed."},
- {"value", (uint64_t) data["n_prompt_tokens_processed_total"]}
- }, {
- {"name", "prompt_seconds_total"},
- {"help", "Prompt process time"},
- {"value", (uint64_t) data["t_prompt_processing_total"] / 1.e3}
- }, {
- {"name", "tokens_predicted_total"},
- {"help", "Number of generation tokens processed."},
- {"value", (uint64_t) data["n_tokens_predicted_total"]}
- }, {
- {"name", "tokens_predicted_seconds_total"},
- {"help", "Predict process time"},
- {"value", (uint64_t) data["t_tokens_generation_total"] / 1.e3}
- }}},
- {"gauge", {{
- {"name", "prompt_tokens_seconds"},
- {"help", "Average prompt throughput in tokens/s."},
- {"value", n_prompt_tokens_processed ? 1.e3 / t_prompt_processing * n_prompt_tokens_processed : 0.}
- },{
- {"name", "predicted_tokens_seconds"},
- {"help", "Average generation throughput in tokens/s."},
- {"value", n_tokens_predicted ? 1.e3 / t_tokens_generation * n_tokens_predicted : 0.}
- },{
- {"name", "kv_cache_usage_ratio"},
- {"help", "KV-cache usage. 1 means 100 percent usage."},
- {"value", 1. * kv_cache_used_cells / params.n_ctx}
- },{
- {"name", "kv_cache_tokens"},
- {"help", "KV-cache tokens."},
- {"value", (uint64_t) data["kv_cache_tokens_count"]}
- },{
- {"name", "requests_processing"},
- {"help", "Number of request processing."},
- {"value", (uint64_t) data["processing"]}
- },{
- {"name", "requests_deferred"},
- {"help", "Number of request deferred."},
- {"value", (uint64_t) data["deferred"]}
- }}}
- };
-
- std::stringstream prometheus;
-
- for (const auto & el : all_metrics_def.items()) {
- const auto & type = el.key();
- const auto & metrics_def = el.value();
-
- for (const auto & metric_def : metrics_def) {
- const std::string name = metric_def["name"];
- const std::string help = metric_def["help"];
-
- auto value = json_value(metric_def, "value", 0.);
- prometheus << "# HELP llamacpp:" << name << " " << help << "\n"
- << "# TYPE llamacpp:" << name << " " << type << "\n"
- << "llamacpp:" << name << " " << value << "\n";
- }
- }
-
- const int64_t t_start = data["t_start"];
- res.set_header("Process-Start-Time-Unix", std::to_string(t_start));
-
- res.set_content(prometheus.str(), "text/plain; version=0.0.4");
- res.status = 200; // HTTP OK
- };
-
- const auto handle_props = [&ctx_server](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
- json data = {
- { "user_name", ctx_server.name_user.c_str() },
- { "assistant_name", ctx_server.name_assistant.c_str() },
- { "default_generation_settings", ctx_server.default_generation_settings_for_props },
- { "total_slots", ctx_server.params.n_parallel }
- };
-
- res.set_content(data.dump(), "application/json; charset=utf-8");
- };
-
- const auto handle_completions = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
-
- json data = json::parse(req.body);
-
- const int id_task = ctx_server.queue_tasks.get_new_id();
-
- ctx_server.queue_results.add_waiting_task_id(id_task);
- ctx_server.request_completion(id_task, -1, data, false, false);
-
- if (!json_value(data, "stream", false)) {
- server_task_result result = ctx_server.queue_results.recv(id_task);
- if (!result.error && result.stop) {
- res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");
- } else {
- res_error(res, result.data);
- }
-
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- } else {
- const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) {
- while (true) {
- server_task_result result = ctx_server.queue_results.recv(id_task);
- if (!result.error) {
- const std::string str =
- "data: " +
- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +
- "\n\n";
-
- LOG_VERBOSE("data stream", {
- { "to_send", str }
- });
-
- if (!sink.write(str.c_str(), str.size())) {
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- return false;
- }
-
- if (result.stop) {
- break;
+ llama.queue_results.remove_waiting_task_id(task_id);
+ } else {
+ const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink)
+ {
+ while (true)
+ {
+ task_result result = llama.queue_results.recv(task_id);
+ if (!result.error) {
+ const std::string str =
+ "data: " +
+ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +
+ "\n\n";
+ LOG_VERBOSE("data stream", {
+ { "to_send", str }
+ });
+ if (!sink.write(str.c_str(), str.size()))
+ {
+ llama.queue_results.remove_waiting_task_id(task_id);
+ return false;
+ }
+ if (result.stop) {
+ break;
+ }
+ } else {
+ const std::string str =
+ "error: " +
+ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +
+ "\n\n";
+ LOG_VERBOSE("data stream", {
+ { "to_send", str }
+ });
+ if (!sink.write(str.c_str(), str.size()))
+ {
+ llama.queue_results.remove_waiting_task_id(task_id);
+ return false;
+ }
+ break;
+ }
}
- } else {
- const std::string str =
- "error: " +
- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +
- "\n\n";
- LOG_VERBOSE("data stream", {
- { "to_send", str }
- });
+ llama.queue_results.remove_waiting_task_id(task_id);
+ sink.done();
+ return true;
+ };
- if (!sink.write(str.c_str(), str.size())) {
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- return false;
- }
+ auto on_complete = [task_id, &llama] (bool)
+ {
+ // cancel
+ llama.request_cancel(task_id);
+ llama.queue_results.remove_waiting_task_id(task_id);
+ };
- break;
- }
+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
}
+ });
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- sink.done();
-
- return true;
- };
-
- auto on_complete = [id_task, &ctx_server] (bool) {
- // cancel
- ctx_server.request_cancel(id_task);
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- };
-
- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
- }
- };
-
- const auto handle_models = [&params, &model_meta](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
-
- json models = {
- {"object", "list"},
- {"data", {
- {
- {"id", params.model_alias},
- {"object", "model"},
- {"created", std::time(0)},
- {"owned_by", "llamacpp"},
- {"meta", model_meta}
- },
- }}
- };
+ svr.Get("/v1/models", [&params](const httplib::Request& req, httplib::Response& res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ std::time_t t = std::time(0);
+
+ json models = {
+ {"object", "list"},
+ {"data", {
+ {
+ {"id", params.model_alias},
+ {"object", "model"},
+ {"created", t},
+ {"owned_by", "llamacpp"}
+ },
+ }}
+ };
- res.set_content(models.dump(), "application/json; charset=utf-8");
- };
+ res.set_content(models.dump(), "application/json; charset=utf-8");
+ });
- const auto handle_chat_completions = [&ctx_server, &sparams, &res_error](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
- json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), sparams.chat_template);
- const int id_task = ctx_server.queue_tasks.get_new_id();
+ // TODO: add mount point without "/v1" prefix -- how?
+ svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ if (!validate_api_key(req, res)) {
+ return;
+ }
+ json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template);
- ctx_server.queue_results.add_waiting_task_id(id_task);
- ctx_server.request_completion(id_task, -1, data, false, false);
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, data, false, false, -1);
- const auto completion_id = gen_chatcmplid();
- if (!json_value(data, "stream", false)) {
- server_task_result result = ctx_server.queue_results.recv(id_task);
+ if (!json_value(data, "stream", false)) {
+ std::string completion_text;
+ task_result result = llama.queue_results.recv(task_id);
- if (!result.error && result.stop) {
- json result_oai = format_final_response_oaicompat(data, result.data, completion_id);
+ if (!result.error && result.stop) {
+ json oaicompat_result = format_final_response_oaicompat(data, result);
- res.set_content(result_oai.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");
- } else {
- res_error(res, result.data);
- }
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- } else {
- const auto chunked_content_provider = [id_task, &ctx_server, completion_id](size_t, httplib::DataSink & sink) {
- while (true) {
- server_task_result result = ctx_server.queue_results.recv(id_task);
- if (!result.error) {
- std::vector<json> result_array = format_partial_response_oaicompat(result.data, completion_id);
-
- for (auto it = result_array.begin(); it != result_array.end(); ++it) {
- if (!it->empty()) {
+ res.set_content(oaicompat_result.dump(-1, ' ', false,
+ json::error_handler_t::replace),
+ "application/json; charset=utf-8");
+ } else {
+ res.status = 500;
+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");
+ }
+ llama.queue_results.remove_waiting_task_id(task_id);
+ } else {
+ const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) {
+ while (true) {
+ task_result llama_result = llama.queue_results.recv(task_id);
+ if (!llama_result.error) {
+ std::vector<json> result_array = format_partial_response_oaicompat( llama_result);
+
+ for (auto it = result_array.begin(); it != result_array.end(); ++it)
+ {
+ if (!it->empty()) {
+ const std::string str =
+ "data: " +
+ it->dump(-1, ' ', false, json::error_handler_t::replace) +
+ "\n\n";
+ LOG_VERBOSE("data stream", {{"to_send", str}});
+ if (!sink.write(str.c_str(), str.size())) {
+ llama.queue_results.remove_waiting_task_id(task_id);
+ return false;
+ }
+ }
+ }
+ if (llama_result.stop) {
+ break;
+ }
+ } else {
const std::string str =
- "data: " +
- it->dump(-1, ' ', false, json::error_handler_t::replace) +
+ "error: " +
+ llama_result.result_json.dump(-1, ' ', false,
+ json::error_handler_t::replace) +
"\n\n";
LOG_VERBOSE("data stream", {{"to_send", str}});
if (!sink.write(str.c_str(), str.size())) {
- ctx_server.queue_results.remove_waiting_task_id(id_task);
+ llama.queue_results.remove_waiting_task_id(task_id);
return false;
}
+ break;
}
}
- if (result.stop) {
- break;
- }
- } else {
- const std::string str =
- "error: " +
- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +
- "\n\n";
- LOG_VERBOSE("data stream", {{"to_send", str}});
- if (!sink.write(str.c_str(), str.size())) {
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- return false;
- }
- break;
- }
- }
- sink.done();
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- return true;
- };
-
- auto on_complete = [id_task, &ctx_server](bool) {
- // cancel request
- ctx_server.request_cancel(id_task);
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- };
-
- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
- }
- };
-
- const auto handle_infill = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
-
- json data = json::parse(req.body);
-
- const int id_task = ctx_server.queue_tasks.get_new_id();
-
- ctx_server.queue_results.add_waiting_task_id(id_task);
- ctx_server.request_completion(id_task, -1, data, true, false);
-
- if (!json_value(data, "stream", false)) {
- server_task_result result = ctx_server.queue_results.recv(id_task);
- if (!result.error && result.stop) {
- res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");
- } else {
- res_error(res, result.data);
- }
-
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- } else {
- const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) {
- while (true) {
- server_task_result result = ctx_server.queue_results.recv(id_task);
- if (!result.error) {
- const std::string str =
- "data: " +
- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +
- "\n\n";
-
- LOG_VERBOSE("data stream", {
- { "to_send", str }
- });
+ sink.done();
+ llama.queue_results.remove_waiting_task_id(task_id);
+ return true;
+ };
- if (!sink.write(str.c_str(), str.size())) {
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- return false;
- }
+ auto on_complete = [task_id, &llama](bool) {
+ // cancel request
+ llama.request_cancel(task_id);
+ llama.queue_results.remove_waiting_task_id(task_id);
+ };
- if (result.stop) {
- break;
- }
- } else {
- break;
- }
+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
}
+ });
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- sink.done();
-
- return true;
- };
-
- auto on_complete = [id_task, &ctx_server] (bool) {
- ctx_server.request_cancel(id_task);
- };
-
- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
- }
- };
-
- const auto handle_tokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
- const json body = json::parse(req.body);
+ svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ if (!validate_api_key(req, res)) {
+ return;
+ }
+ json data = json::parse(req.body);
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, data, true, false, -1);
+ if (!json_value(data, "stream", false)) {
+ std::string completion_text;
+ task_result result = llama.queue_results.recv(task_id);
+ if (!result.error && result.stop)
+ {
+ res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");
+ }
+ else
+ {
+ res.status = 404;
+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");
+ }
+ llama.queue_results.remove_waiting_task_id(task_id);
+ } else {
+ const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink) {
+ while (true)
+ {
+ task_result result = llama.queue_results.recv(task_id);
+ if (!result.error) {
+ const std::string str =
+ "data: " +
+ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +
+ "\n\n";
+ LOG_VERBOSE("data stream", {
+ { "to_send", str }
+ });
+ if (!sink.write(str.c_str(), str.size()))
+ {
+ llama.queue_results.remove_waiting_task_id(task_id);
+ return false;
+ }
+ if (result.stop)
+ {
+ break;
+ }
+ }
+ else
+ {
+ break;
+ }
+ }
- std::vector<llama_token> tokens;
- if (body.count("content") != 0) {
- tokens = ctx_server.tokenize(body["content"], false);
- }
- const json data = format_tokenizer_response(tokens);
- return res.set_content(data.dump(), "application/json; charset=utf-8");
- };
+ llama.queue_results.remove_waiting_task_id(task_id);
+ sink.done();
+ return true;
+ };
- const auto handle_detokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
- const json body = json::parse(req.body);
+ auto on_complete = [task_id, &llama] (bool)
+ {
+ // cancel
+ llama.request_cancel(task_id);
+ };
- std::string content;
- if (body.count("tokens") != 0) {
- const std::vector<llama_token> tokens = body["tokens"];
- content = tokens_to_str(ctx_server.ctx, tokens.cbegin(), tokens.cend());
- }
+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
+ }
+ });
- const json data = format_detokenized_response(content);
- return res.set_content(data.dump(), "application/json; charset=utf-8");
- };
+ svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res)
+ { return res.set_content("", "application/json; charset=utf-8"); });
- const auto handle_embeddings = [&params, &ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
- if (!params.embedding) {
- res.status = 501;
- res.set_content("This server does not support embeddings. Start it with `--embeddings`", "text/plain; charset=utf-8");
- return;
- }
+ svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ const json body = json::parse(req.body);
+ std::vector<llama_token> tokens;
+ if (body.count("content") != 0)
+ {
+ tokens = llama.tokenize(body["content"], false);
+ }
+ const json data = format_tokenizer_response(tokens);
+ return res.set_content(data.dump(), "application/json; charset=utf-8");
+ });
- const json body = json::parse(req.body);
- bool is_openai = false;
+ svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ const json body = json::parse(req.body);
+ std::string content;
+ if (body.count("tokens") != 0)
+ {
+ const std::vector<llama_token> tokens = body["tokens"];
+ content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());
+ }
- // an input prompt can be a string or a list of tokens (integer)
- json prompt;
- if (body.count("input") != 0) {
- is_openai = true;
- prompt = body["input"];
- } else if (body.count("content") != 0) {
- // with "content", we only support single prompt
- prompt = std::vector<std::string>{body["content"]};
- } else {
- res_error(res, format_error_response("\"input\" or \"content\" must be provided", ERROR_TYPE_INVALID_REQUEST));
- return;
- }
+ const json data = format_detokenized_response(content);
+ return res.set_content(data.dump(), "application/json; charset=utf-8");
+ });
- // create and queue the task
- json responses;
- {
- const int id_task = ctx_server.queue_tasks.get_new_id();
- ctx_server.queue_results.add_waiting_task_id(id_task);
- ctx_server.request_completion(id_task, -1, {{"prompt", prompt}}, false, true);
+ svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ const json body = json::parse(req.body);
+ json prompt;
+ if (body.count("content") != 0)
+ {
+ prompt = body["content"];
+ }
+ else
+ {
+ prompt = "";
+ }
- // get the result
- server_task_result result = ctx_server.queue_results.recv(id_task);
- ctx_server.queue_results.remove_waiting_task_id(id_task);
- if (!result.error) {
- if (result.data.count("results")) {
- // result for multi-task
- responses = result.data["results"];
- } else {
- // result for single task
- responses = std::vector<json>{result.data};
+ json image_data;
+ if (body.count("image_data") != 0) {
+ image_data = body["image_data"];
+ }
+ else
+ {
+ image_data = "";
}
- } else {
- // error received, ignore everything else
- res_error(res, result.data);
- return;
- }
- }
- // write JSON response
- json root = is_openai
- ? format_embeddings_response_oaicompat(body, responses)
- : responses[0];
- return res.set_content(root.dump(), "application/json; charset=utf-8");
- };
+ // create and queue the task
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1);
- auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) {
- return [content, len, mime_type](const httplib::Request &, httplib::Response & res) {
- res.set_content(reinterpret_cast<const char*>(content), len, mime_type);
- return false;
- };
- };
+ // get the result
+ task_result result = llama.queue_results.recv(task_id);
+ llama.queue_results.remove_waiting_task_id(task_id);
- //
- // Router
- //
+ // send the result
+ return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");
+ });
- // register static assets routes
- if (!sparams.public_path.empty()) {
- // Set the base directory for serving static files
- svr->set_base_dir(sparams.public_path);
- }
+ svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res)
+ {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ const json body = json::parse(req.body);
- // register API routes
- svr->Get ("/health", handle_health);
- svr->Get ("/slots", handle_slots);
- svr->Get ("/metrics", handle_metrics);
- svr->Get ("/props", handle_props);
- svr->Get ("/v1/models", handle_models);
- svr->Post("/completion", handle_completions); // legacy
- svr->Post("/completions", handle_completions);
- svr->Post("/v1/completions", handle_completions);
- svr->Post("/chat/completions", handle_chat_completions);
- svr->Post("/v1/chat/completions", handle_chat_completions);
- svr->Post("/infill", handle_infill);
- svr->Post("/embedding", handle_embeddings); // legacy
- svr->Post("/embeddings", handle_embeddings);
- svr->Post("/v1/embeddings", handle_embeddings);
- svr->Post("/tokenize", handle_tokenize);
- svr->Post("/detokenize", handle_detokenize);
-
- //
- // Start the server
- //
- if (sparams.n_threads_http < 1) {
- // +2 threads for monitoring endpoints
- sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1);
- }
- log_data["n_threads_http"] = std::to_string(sparams.n_threads_http);
- svr->new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); };
+ json prompt;
+ if (body.count("input") != 0)
+ {
+ prompt = body["input"];
+ // batch
+ if(prompt.is_array()) {
+ json data = json::array();
+ int i = 0;
+ for (const json &elem : prompt) {
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, { {"prompt", elem}, { "n_predict", 0} }, false, true, -1);
+
+ // get the result
+ task_result result = llama.queue_results.recv(task_id);
+ llama.queue_results.remove_waiting_task_id(task_id);
+
+ json embedding = json{
+ {"embedding", json_value(result.result_json, "embedding", json::array())},
+ {"index", i++},
+ {"object", "embedding"}
+ };
+ data.push_back(embedding);
+ }
+ json result = format_embeddings_response_oaicompat(body, data);
+ return res.set_content(result.dump(), "application/json; charset=utf-8");
+ }
+ }
+ else
+ {
+ prompt = "";
+ }
- LOG_INFO("HTTP server listening", log_data);
+ // create and queue the task
+ const int task_id = llama.queue_tasks.get_new_id();
+ llama.queue_results.add_waiting_task_id(task_id);
+ llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}}, false, true, -1);
- // launch browser tab
- if (!sparams.nobrowser) {
- char url[128];
- snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);
- llamafile_launch_browser(url);
- }
+ // get the result
+ task_result result = llama.queue_results.recv(task_id);
+ llama.queue_results.remove_waiting_task_id(task_id);
- if (!FLAG_unsecure) {
- if (IsXnu()) {
- // Cosmopolitan libc explicitly does not support cosmo_dlopen on x64
- // macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt
- // to enable sandboxing and ignore the failure on x64 macOS.
- // (Alternatively, the sandbox-exec(1) command could be used to
- // launch llamafile with a sandbox profile.) A goal of llamafile is
- // to work easily out-of-the-box and it already operates without a
- // sandbox on macOS so don't require --unsecure on x64 macOS to
- // workaround the failure. If support is added later, it will be
- // used without further changes. On Apple Silicon, if sandbox
- // initialization fails, exit and recommend using --unsecure to
- // disable sandboxing.
- std::string error_msg;
- int rv = mac_sandbox_init(error_msg);
- if (rv != 0) {
- fprintf(stderr, "Sandbox initialization failed:\n"
- "rv: %d, error: %s.\n"
- "Disable sandboxing with the --unsecure option.\n",
- rv, error_msg.c_str());
- if (IsXnuSilicon()) {
- exit(1);
- } else {
- LOG_WARNING("Sandbox initialization failed. Not supported "
- "on x64 macOS. Continuing.", {});
- }
- }
- } else if (!llamafile_has_gpu()) {
- printf("In the sandboxing block!\n");
- // Enables pledge() security on Linux and OpenBSD.
- // - We do this *after* binding the server socket.
- // - We do this *after* opening the log file for writing.
- // - We do this *after* opening a tab in the user browser.
- // - We do this *before* loading any weights or graphdefs.
- // In effect, what this does is:
- // - Filesystem access is disabled entirely (except ZipOS).
- // - On Linux, network access is restricted to accept() only.
- // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.
- char promises[32];
- if (IsOpenbsd()) {
- strlcpy(promises, "stdio inet", sizeof(promises));
- } else {
- strlcpy(promises, "stdio anet", sizeof(promises));
- }
- if (!startswith(sparams.public_path.c_str(), "/zip/")) {
- strlcat(promises, " rpath", sizeof(promises));
- }
- __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;
- if (pledge(0, 0)) {
- LOG_TEE("warning: this OS doesn't support pledge() security\n");
- } else if (pledge(promises, 0)) {
- perror("pledge");
- exit(1);
- }
- }
- }
+ json data = json::array({json{
+ {"embedding", json_value(result.result_json, "embedding", json::array())},
+ {"index", 0},
+ {"object", "embedding"}
+ }}
+ );
- // run the HTTP server in a thread - see comment below
- std::thread t([&]() {
- if (!svr->listen_after_bind()) {
- state.store(SERVER_STATE_ERROR);
- return 1;
- }
+ json root = format_embeddings_response_oaicompat(body, data);
- return 0;
- });
+ // send the result
+ return res.set_content(root.dump(), "application/json; charset=utf-8");
+ });
- ctx_server.queue_tasks.on_new_task(std::bind(
- &server_context::process_single_task, &ctx_server, std::placeholders::_1));
- ctx_server.queue_tasks.on_finish_multitask(std::bind(
- &server_context::on_finish_multitask, &ctx_server, std::placeholders::_1));
- ctx_server.queue_tasks.on_update_slots(std::bind(
- &server_context::update_slots, &ctx_server));
- ctx_server.queue_results.on_multitask_update(std::bind(
- &server_queue::update_multitask,
- &ctx_server.queue_tasks,
+ // GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!?
+ // "Bus error: 10" - this is on macOS, it does not crash on Linux
+ //std::thread t2([&]()
+ /*{
+ bool running = true;
+ while (running)
+ {
+ running = llama.update_slots();
+ }
+ }*/
+ //);
+
+ llama.queue_tasks.on_new_task(std::bind(
+ &llama_server_context::process_single_task, &llama, std::placeholders::_1));
+ llama.queue_tasks.on_finish_multitask(std::bind(
+ &llama_server_context::on_finish_multitask, &llama, std::placeholders::_1));
+ llama.queue_tasks.on_all_tasks_finished(std::bind(
+ &llama_server_context::run_on_all_tasks_finished, &llama));
+ llama.queue_results.on_multitask_update(std::bind(
+ &llama_server_queue::update_multitask,
+ &llama.queue_tasks,
std::placeholders::_1,
std::placeholders::_2,
std::placeholders::_3
));
shutdown_handler = [&](int) {
- ctx_server.queue_tasks.terminate();
+ llama.queue_tasks.terminate();
};
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
@@ -3669,13 +3635,10 @@ int server_cli(int argc, char ** argv) {
};
SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
#endif
-
- ctx_server.queue_tasks.start_loop();
-
- svr->stop();
+ llama.queue_tasks.start_loop();
+ svr.stop();
t.join();
llama_backend_free();
-
return 0;
}
diff --git a/llama.cpp/server/utils.h b/llama.cpp/server/utils.h
index b84cf70e..ca9b7bdc 100644
--- a/llama.cpp/server/utils.h
+++ b/llama.cpp/server/utils.h
@@ -1,31 +1,20 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#pragma once
-#include "llama.cpp/llama.h"
-#include "llama.cpp/common.h"
-#include "llama.cpp/json.h"
+#pragma once
#include <string>
#include <vector>
-#include <sstream>
-#include <random>
+#include <set>
+#include <mutex>
+#include <condition_variable>
+#include <unordered_map>
-#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"
+#include "llama.cpp/json.h"
+#include "llama.cpp/llava/clip.h"
using json = nlohmann::json;
-// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/11
-enum error_type {
- ERROR_TYPE_INVALID_REQUEST,
- ERROR_TYPE_AUTHENTICATION,
- ERROR_TYPE_SERVER,
- ERROR_TYPE_NOT_FOUND,
- ERROR_TYPE_PERMISSION,
- ERROR_TYPE_UNAVAILABLE, // custom error
- ERROR_TYPE_NOT_SUPPORTED, // custom error
-};
-
extern bool server_verbose;
extern bool server_log_json;
@@ -50,35 +39,125 @@ extern bool server_log_json;
#define LOG_WARNING(MSG, ...) server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__)
#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__)
-template <typename T>
-static T json_value(const json &body, const std::string &key, const T &default_value) {
- // Fallback null to default value
- return body.contains(key) && !body.at(key).is_null()
- ? body.value(key, default_value)
- : default_value;
-}
+//
+// parallel
+//
+
+enum server_state {
+ SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet
+ SERVER_STATE_READY, // Server is ready and model is loaded
+ SERVER_STATE_ERROR // An error occurred, load_model failed
+};
+
+enum task_type {
+ TASK_TYPE_COMPLETION,
+ TASK_TYPE_CANCEL,
+ TASK_TYPE_NEXT_RESPONSE,
+ TASK_TYPE_METRICS
+};
+
+struct task_server {
+ int id = -1; // to be filled by llama_server_queue
+ int target_id;
+ task_type type;
+ json data;
+ bool infill_mode = false;
+ bool embedding_mode = false;
+ int multitask_id = -1;
+};
+
+struct task_result {
+ int id;
+ int multitask_id = -1;
+ bool stop;
+ bool error;
+ json result_json;
+};
+
+struct task_multi {
+ int id;
+ std::set<int> subtasks_remaining{};
+ std::vector<task_result> results{};
+};
+
+// TODO: can become bool if we can't find use of more states
+enum slot_state
+{
+ IDLE,
+ PROCESSING,
+};
+
+enum slot_command
+{
+ NONE,
+ LOAD_PROMPT,
+ RELEASE,
+};
+
+struct slot_params
+{
+ bool stream = true;
+ bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt
+
+ uint32_t seed = -1; // RNG seed
+ int32_t n_keep = 0; // number of tokens to keep from initial prompt
+ int32_t n_predict = -1; // new tokens to predict
-static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) {
+ std::vector<std::string> antiprompt;
+
+ json input_prefix;
+ json input_suffix;
+};
+
+struct slot_image
+{
+ int32_t id;
+
+ bool request_encode_image = false;
+ float * image_embedding = nullptr;
+ int32_t image_tokens = 0;
+
+ clip_image_u8 * img_data;
+
+ std::string prefix_prompt; // before of this image
+};
+
+// completion token output with probabilities
+struct completion_token_output
+{
+ struct token_prob
+ {
+ llama_token tok;
+ float prob;
+ };
+
+ std::vector<token_prob> probs;
+ llama_token tok;
+ std::string text_to_send;
+};
+
+static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra)
+{
std::stringstream ss_tid;
ss_tid << std::this_thread::get_id();
json log = nlohmann::ordered_json{
- {"tid", ss_tid.str()},
+ {"tid", ss_tid.str()},
{"timestamp", time(nullptr)},
};
if (server_log_json) {
- log.merge_patch( {
- {"level", level},
- {"function", function},
- {"line", line},
- {"msg", message},
- });
-
+ log.merge_patch(
+ {
+ {"level", level},
+ {"function", function},
+ {"line", line},
+ {"msg", message},
+ });
if (!extra.empty()) {
log.merge_patch(extra);
}
- printf("%s\n", log.dump(-1, ' ', false, json::error_handler_t::replace).c_str());
+ std::cout << log.dump(-1, ' ', false, json::error_handler_t::replace) << "\n" << std::flush;
} else {
char buf[1024];
snprintf(buf, 1024, "%4s [%24s] %s", level, function, message);
@@ -91,7 +170,8 @@ static inline void server_log(const char *level, const char *function, int line,
for (const auto& el : log.items())
{
const std::string value = el.value().dump(-1, ' ', false, json::error_handler_t::replace);
- ss << " " << el.key() << "=" << value;
+ snprintf(buf, 1024, " %s=%s", el.key().c_str(), value.c_str());
+ ss << buf;
}
const std::string str = ss.str();
@@ -101,25 +181,36 @@ static inline void server_log(const char *level, const char *function, int line,
}
//
-// chat template utils
+// server utils
//
+template <typename T>
+static T json_value(const json &body, const std::string &key, const T &default_value)
+{
+ // Fallback null to default value
+ return body.contains(key) && !body.at(key).is_null()
+ ? body.value(key, default_value)
+ : default_value;
+}
+
// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid
inline bool verify_custom_template(const std::string & tmpl) {
llama_chat_message chat[] = {{"user", "test"}};
- int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, nullptr, 0);
+ std::vector<char> buf(1);
+ int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, buf.data(), buf.size());
return res >= 0;
}
// Format given chat. If tmpl is empty, we take the template from model metadata
-inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages) {
+inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages)
+{
size_t alloc_size = 0;
// vector holding all allocated string to be passed to llama_chat_apply_template
std::vector<std::string> str(messages.size() * 2);
std::vector<llama_chat_message> chat(messages.size());
for (size_t i = 0; i < messages.size(); ++i) {
- const auto & curr_msg = messages[i];
+ auto &curr_msg = messages[i];
str[i*2 + 0] = json_value(curr_msg, "role", std::string(""));
str[i*2 + 1] = json_value(curr_msg, "content", std::string(""));
alloc_size += str[i*2 + 1].length();
@@ -139,13 +230,252 @@ inline std::string format_chat(const struct llama_model * model, const std::stri
res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size());
}
- const std::string formatted_chat(buf.data(), res);
-
+ std::string formatted_chat(buf.data(), res);
LOG_VERBOSE("formatted_chat", {{"text", formatted_chat.c_str()}});
return formatted_chat;
}
+//
+// work queue utils
+//
+
+struct llama_server_queue {
+ int id = 0;
+ std::mutex mutex_tasks;
+ bool running;
+ // queues
+ std::vector<task_server> queue_tasks;
+ std::vector<task_server> queue_tasks_deferred;
+ std::vector<task_multi> queue_multitasks;
+ std::condition_variable condition_tasks;
+ // callback functions
+ std::function<void(task_server&)> callback_new_task;
+ std::function<void(task_multi&)> callback_finish_multitask;
+ std::function<void(void)> callback_all_task_finished;
+
+ // Add a new task to the end of the queue
+ int post(task_server task) {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ if (task.id == -1) {
+ task.id = id++;
+ LOG_VERBOSE("new task id", {{"new_id", task.id}});
+ }
+ queue_tasks.push_back(std::move(task));
+ condition_tasks.notify_one();
+ return task.id;
+ }
+
+ // Add a new task, but defer until one slot is available
+ void defer(task_server task) {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ queue_tasks_deferred.push_back(std::move(task));
+ }
+
+ // Get the next id for creating anew task
+ int get_new_id() {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ int new_id = id++;
+ LOG_VERBOSE("new task id", {{"new_id", new_id}});
+ return new_id;
+ }
+
+ // Register function to process a new task
+ void on_new_task(std::function<void(task_server&)> callback) {
+ callback_new_task = callback;
+ }
+
+ // Register function to process a multitask
+ void on_finish_multitask(std::function<void(task_multi&)> callback) {
+ callback_finish_multitask = callback;
+ }
+
+ // Register the function to be called when the batch of tasks is finished
+ void on_all_tasks_finished(std::function<void(void)> callback) {
+ callback_all_task_finished = callback;
+ }
+
+ // Call when the state of one slot is changed
+ void notify_slot_changed() {
+ // move deferred tasks back to main loop
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ for (auto & task : queue_tasks_deferred) {
+ queue_tasks.push_back(std::move(task));
+ }
+ queue_tasks_deferred.clear();
+ }
+
+ // end the start_loop routine
+ void terminate() {
+ {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ running = false;
+ }
+ condition_tasks.notify_all();
+ }
+
+ // Start the main loop.
+ void start_loop() {
+ running = true;
+ while (true) {
+ LOG_VERBOSE("new task may arrive", {});
+ {
+ while (true)
+ {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ if (queue_tasks.empty()) {
+ lock.unlock();
+ break;
+ }
+ task_server task = queue_tasks.front();
+ queue_tasks.erase(queue_tasks.begin());
+ lock.unlock();
+ LOG_VERBOSE("callback_new_task", {{"task_id", task.id}});
+ callback_new_task(task);
+ }
+ LOG_VERBOSE("callback_all_task_finished", {});
+ // process and update all the multitasks
+ auto queue_iterator = queue_multitasks.begin();
+ while (queue_iterator != queue_multitasks.end())
+ {
+ if (queue_iterator->subtasks_remaining.empty())
+ {
+ // all subtasks done == multitask is done
+ task_multi current_multitask = *queue_iterator;
+ callback_finish_multitask(current_multitask);
+ // remove this multitask
+ queue_iterator = queue_multitasks.erase(queue_iterator);
+ }
+ else
+ {
+ ++queue_iterator;
+ }
+ }
+ // all tasks in the current loop is finished
+ callback_all_task_finished();
+ }
+ LOG_VERBOSE("wait for new task", {});
+ // wait for new task
+ {
+ std::unique_lock<std::mutex> lock(mutex_tasks);
+ if (queue_tasks.empty()) {
+ if (!running) {
+ LOG_VERBOSE("ending start_loop", {});
+ return;
+ }
+ condition_tasks.wait(lock, [&]{
+ return (!queue_tasks.empty() || !running);
+ });
+ }
+ }
+ }
+ }
+
+ //
+ // functions to manage multitasks
+ //
+
+ // add a multitask by specifying the id of all subtask (subtask is a task_server)
+ void add_multitask(int multitask_id, std::vector<int>& sub_ids)
+ {
+ std::lock_guard<std::mutex> lock(mutex_tasks);
+ task_multi multi;
+ multi.id = multitask_id;
+ std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));
+ queue_multitasks.push_back(multi);
+ }
+
+ // updatethe remaining subtasks, while appending results to multitask
+ void update_multitask(int multitask_id, int subtask_id, task_result& result)
+ {
+ std::lock_guard<std::mutex> lock(mutex_tasks);
+ for (auto& multitask : queue_multitasks)
+ {
+ if (multitask.id == multitask_id)
+ {
+ multitask.subtasks_remaining.erase(subtask_id);
+ multitask.results.push_back(result);
+ }
+ }
+ }
+};
+
+struct llama_server_response {
+ typedef std::function<void(int, int, task_result&)> callback_multitask_t;
+ callback_multitask_t callback_update_multitask;
+ // for keeping track of all tasks waiting for the result
+ std::set<int> waiting_task_ids;
+ // the main result queue
+ std::vector<task_result> queue_results;
+ std::mutex mutex_results;
+ std::condition_variable condition_results;
+
+ void add_waiting_task_id(int task_id) {
+ LOG_VERBOSE("waiting for task id", {{"task_id", task_id}});
+ std::unique_lock<std::mutex> lock(mutex_results);
+ waiting_task_ids.insert(task_id);
+ }
+
+ void remove_waiting_task_id(int task_id) {
+ LOG_VERBOSE("remove waiting for task id", {{"task_id", task_id}});
+ std::unique_lock<std::mutex> lock(mutex_results);
+ waiting_task_ids.erase(task_id);
+ }
+
+ // This function blocks the thread until there is a response for this task_id
+ task_result recv(int task_id) {
+ while (true)
+ {
+ std::unique_lock<std::mutex> lock(mutex_results);
+ condition_results.wait(lock, [&]{
+ return !queue_results.empty();
+ });
+
+ for (int i = 0; i < (int) queue_results.size(); i++)
+ {
+ if (queue_results[i].id == task_id)
+ {
+ assert(queue_results[i].multitask_id == -1);
+ task_result res = queue_results[i];
+ queue_results.erase(queue_results.begin() + i);
+ return res;
+ }
+ }
+ }
+
+ // should never reach here
+ }
+
+ // Register the function to update multitask
+ void on_multitask_update(callback_multitask_t callback) {
+ callback_update_multitask = callback;
+ }
+
+ // Send a new result to a waiting task_id
+ void send(task_result result) {
+ std::unique_lock<std::mutex> lock(mutex_results);
+ LOG_VERBOSE("send new result", {{"task_id", result.id}});
+ for (auto& task_id : waiting_task_ids) {
+ // LOG_TEE("waiting task id %i \n", task_id);
+ // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result
+ if (result.multitask_id == task_id)
+ {
+ LOG_VERBOSE("callback_update_multitask", {{"task_id", task_id}});
+ callback_update_multitask(task_id, result.id, result);
+ continue;
+ }
+
+ if (result.id == task_id)
+ {
+ LOG_VERBOSE("queue_results.push_back", {{"task_id", task_id}});
+ queue_results.push_back(result);
+ condition_results.notify_all();
+ return;
+ }
+ }
+ }
+};
+
//
// base64 utils (TODO: move to common in the future)
//
@@ -155,11 +485,13 @@ static const std::string base64_chars =
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
-static inline bool is_base64(uint8_t c) {
+static inline bool is_base64(uint8_t c)
+{
return (isalnum(c) || (c == '+') || (c == '/'));
}
-static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string) {
+static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string)
+{
int i = 0;
int j = 0;
int in_ = 0;
@@ -171,10 +503,13 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_str
std::vector<uint8_t> ret;
- while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {
+ while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))
+ {
char_array_4[i++] = encoded_string[in_]; in_++;
- if (i == 4) {
- for (i = 0; i < 4; i++) {
+ if (i == 4)
+ {
+ for (i = 0; i <4; i++)
+ {
char_array_4[i] = base64_chars.find(char_array_4[i]);
}
@@ -182,20 +517,23 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_str
char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
- for (i = 0; (i < 3); i++) {
+ for (i = 0; (i < 3); i++)
+ {
ret.push_back(char_array_3[i]);
}
-
i = 0;
}
}
- if (i) {
- for (j = i; j < 4; j++) {
+ if (i)
+ {
+ for (j = i; j <4; j++)
+ {
char_array_4[j] = 0;
}
- for (j = 0; j < 4; j++) {
+ for (j = 0; j <4; j++)
+ {
char_array_4[j] = base64_chars.find(char_array_4[j]);
}
@@ -203,7 +541,8 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_str
char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
- for (j = 0; j < i - 1; j++) {
+ for (j = 0; (j < i - 1); j++)
+ {
ret.push_back(char_array_3[j]);
}
}
@@ -215,7 +554,8 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_str
// random string / id
//
-static std::string random_string() {
+static std::string random_string()
+{
static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
std::random_device rd;
@@ -230,389 +570,9 @@ static std::string random_string() {
return result;
}
-static std::string gen_chatcmplid() {
+static std::string gen_chatcmplid()
+{
std::stringstream chatcmplid;
chatcmplid << "chatcmpl-" << random_string();
-
return chatcmplid.str();
}
-
-//
-// other common utils
-//
-
-static size_t common_part(const std::vector<llama_token> & a, const std::vector<llama_token> & b) {
- size_t i;
- for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {}
-
- return i;
-}
-
-static bool ends_with(const std::string & str, const std::string & suffix) {
- return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
-}
-
-static size_t find_partial_stop_string(const std::string &stop, const std::string &text) {
- if (!text.empty() && !stop.empty()) {
- const char text_last_char = text.back();
- for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) {
- if (stop[char_index] == text_last_char) {
- const std::string current_partial = stop.substr(0, char_index + 1);
- if (ends_with(text, current_partial)) {
- return text.size() - char_index - 1;
- }
- }
- }
- }
-
- return std::string::npos;
-}
-
-// TODO: reuse llama_detokenize
-template <class Iter>
-static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) {
- std::string ret;
- for (; begin != end; ++begin) {
- ret += llama_token_to_piece(ctx, *begin);
- }
-
- return ret;
-}
-
-// format incomplete utf-8 multibyte character for output
-static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {
- std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token);
-
- // if the size is 1 and first bit is 1, meaning it's a partial character
- // (size > 1 meaning it's already a known token)
- if (out.size() == 1 && (out[0] & 0x80) == 0x80) {
- std::stringstream ss;
- ss << std::hex << (out[0] & 0xff);
- std::string res(ss.str());
- out = "byte: \\x" + res;
- }
-
- return out;
-}
-
-struct completion_token_output {
- llama_token tok;
- std::string text_to_send;
-
- struct token_prob {
- llama_token tok;
- float prob;
- };
-
- std::vector<token_prob> probs;
-};
-
-// convert a vector of completion_token_output to json
-static json probs_vector_to_json(const llama_context * ctx, const std::vector<completion_token_output> & probs) {
- json out = json::array();
-
- for (const auto & prob : probs) {
- json probs_for_token = json::array();
-
- for (const auto & p : prob.probs) {
- const std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);
- probs_for_token.push_back(json {
- {"tok_str", tok_str},
- {"prob", p.prob},
- });
- }
-
- const std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);
- out.push_back(json {
- {"content", tok_str},
- {"probs", probs_for_token},
- });
- }
-
- return out;
-}
-
-//
-// OAI utils
-//
-
-static json oaicompat_completion_params_parse(
- const struct llama_model * model,
- const json & body, /* openai api json semantics */
- const std::string & chat_template) {
- json llama_params;
-
- llama_params["__oaicompat"] = true;
-
- // Map OpenAI parameters to llama.cpp parameters
- //
- // For parameters that are defined by the OpenAI documentation (e.g.
- // temperature), we explicitly specify OpenAI's intended default; we
- // need to do that because sometimes OpenAI disagrees with llama.cpp
- //
- // https://platform.openai.com/docs/api-reference/chat/create
- llama_sampling_params default_sparams;
- llama_params["model"] = json_value(body, "model", std::string("unknown"));
- llama_params["prompt"] = format_chat(model, chat_template, body["messages"]);
- llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);
- llama_params["temperature"] = json_value(body, "temperature", 0.0);
- llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);
- llama_params["top_p"] = json_value(body, "top_p", 1.0);
- llama_params["n_predict"] = json_value(body, "max_tokens", -1);
- llama_params["logit_bias"] = json_value(body, "logit_bias", json::object());
- llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);
- llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);
- llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);
- llama_params["stream"] = json_value(body, "stream", false);
- llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);
- llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);
- llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);
- llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);
- llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);
- llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);
- llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);
- llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);
- llama_params["n_keep"] = json_value(body, "n_keep", 0);
-
- if (body.contains("grammar")) {
- llama_params["grammar"] = json_value(body, "grammar", json::object());
- }
-
- if (body.contains("response_format")) {
- auto response_format = json_value(body, "response_format", json::object());
- if (response_format.contains("type")) {
- if (response_format["type"] == "json_object") {
- llama_params["json_schema"] = json_value(response_format, "schema", json::object());
- } else {
- throw std::runtime_error("response_format type not supported: " + response_format["type"].dump());
- }
- }
- }
-
- // Handle 'stop' field
- if (body.contains("stop") && body["stop"].is_string()) {
- llama_params["stop"] = json::array({body["stop"].get<std::string>()});
- } else {
- llama_params["stop"] = json_value(body, "stop", json::array());
- }
-
- // Ensure there is ChatML-specific end sequence among stop words
- llama_params["stop"].push_back("<|im_end|>");
-
- return llama_params;
-}
-
-static json format_final_response_oaicompat(const json & request, json result, const std::string & completion_id, bool streaming = false) {
- bool stopped_word = result.count("stopped_word") != 0;
- bool stopped_eos = json_value(result, "stopped_eos", false);
- int num_tokens_predicted = json_value(result, "tokens_predicted", 0);
- int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);
- std::string content = json_value(result, "content", std::string(""));
-
- std::string finish_reason = "length";
- if (stopped_word || stopped_eos) {
- finish_reason = "stop";
- }
-
- json choices =
- streaming ? json::array({json{{"finish_reason", finish_reason},
- {"index", 0},
- {"delta", json::object()}}})
- : json::array({json{{"finish_reason", finish_reason},
- {"index", 0},
- {"message", json{{"content", content},
- {"role", "assistant"}}}}});
-
- std::time_t t = std::time(0);
-
- json res = json {
- {"choices", choices},
- {"created", t},
- {"model",
- json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
- {"object", streaming ? "chat.completion.chunk" : "chat.completion"},
- {"usage", json {
- {"completion_tokens", num_tokens_predicted},
- {"prompt_tokens", num_prompt_tokens},
- {"total_tokens", num_tokens_predicted + num_prompt_tokens}
- }},
- {"id", completion_id}
- };
-
- if (server_verbose) {
- res["__verbose"] = result;
- }
-
- if (result.contains("completion_probabilities")) {
- res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());
- }
-
- return res;
-}
-
-// return value is vector as there is one case where we might need to generate two responses
-static std::vector<json> format_partial_response_oaicompat(json result, const std::string & completion_id) {
- if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {
- return std::vector<json>({result});
- }
-
- bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;
- std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));
-
- bool stopped_word = json_value(result, "stopped_word", false);
- bool stopped_eos = json_value(result, "stopped_eos", false);
- bool stopped_limit = json_value(result, "stopped_limit", false);
- std::string content = json_value(result, "content", std::string(""));
-
- std::string finish_reason;
- if (stopped_word || stopped_eos) {
- finish_reason = "stop";
- }
- if (stopped_limit) {
- finish_reason = "length";
- }
-
- std::time_t t = std::time(0);
-
- json choices;
-
- if (!finish_reason.empty()) {
- choices = json::array({json{{"finish_reason", finish_reason},
- {"index", 0},
- {"delta", json::object()}}});
- } else {
- if (first) {
- if (content.empty()) {
- choices = json::array({json{{"finish_reason", nullptr},
- {"index", 0},
- {"delta", json{{"role", "assistant"}}}}});
- } else {
- // We have to send this as two updates to conform to openai behavior
- json initial_ret = json{{"choices", json::array({json{
- {"finish_reason", nullptr},
- {"index", 0},
- {"delta", json{
- {"role", "assistant"}
- }}}})},
- {"created", t},
- {"id", completion_id},
- {"model", modelname},
- {"object", "chat.completion.chunk"}};
-
- json second_ret = json{
- {"choices", json::array({json{{"finish_reason", nullptr},
- {"index", 0},
- {"delta", json{
- {"content", content}}}
- }})},
- {"created", t},
- {"id", completion_id},
- {"model", modelname},
- {"object", "chat.completion.chunk"}};
-
- return std::vector<json>({initial_ret, second_ret});
- }
- } else {
- // Some idiosyncrasy in task processing logic makes several trailing calls
- // with empty content, we ignore these at the calee site.
- if (content.empty()) {
- return std::vector<json>({json::object()});
- }
-
- choices = json::array({json{
- {"finish_reason", nullptr},
- {"index", 0},
- {"delta",
- json{
- {"content", content},
- }},
- }});
- }
- }
-
- json ret = json {
- {"choices", choices},
- {"created", t},
- {"id", completion_id},
- {"model", modelname},
- {"object", "chat.completion.chunk"}
- };
-
- return std::vector<json>({ret});
-}
-
-static json format_embeddings_response_oaicompat(const json & request, const json & embeddings) {
- json data = json::array();
- int i = 0;
- for (auto & elem : embeddings) {
- data.push_back(json{
- {"embedding", json_value(elem, "embedding", json::array())},
- {"index", i++},
- {"object", "embedding"}
- });
- }
-
- json res = json {
- {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
- {"object", "list"},
- {"usage", json {
- {"prompt_tokens", 0},
- {"total_tokens", 0}
- }},
- {"data", data}
- };
-
- return res;
-}
-
-static json format_tokenizer_response(const std::vector<llama_token> & tokens) {
- return json {
- {"tokens", tokens}
- };
-}
-
-static json format_detokenized_response(const std::string & content) {
- return json {
- {"content", content}
- };
-}
-
-static json format_error_response(const std::string & message, const enum error_type type) {
- std::string type_str;
- int code = 500;
- switch (type) {
- case ERROR_TYPE_INVALID_REQUEST:
- type_str = "invalid_request_error";
- code = 400;
- break;
- case ERROR_TYPE_AUTHENTICATION:
- type_str = "authentication_error";
- code = 401;
- break;
- case ERROR_TYPE_NOT_FOUND:
- type_str = "not_found_error";
- code = 404;
- break;
- case ERROR_TYPE_SERVER:
- type_str = "server_error";
- code = 500;
- break;
- case ERROR_TYPE_PERMISSION:
- type_str = "permission_error";
- code = 403;
- break;
- case ERROR_TYPE_NOT_SUPPORTED:
- type_str = "not_supported_error";
- code = 501;
- break;
- case ERROR_TYPE_UNAVAILABLE:
- type_str = "unavailable_error";
- code = 503;
- break;
- }
- return json {
- {"code", code},
- {"message", message},
- {"type", type_str},
- };
-}
diff --git a/llamafile/kernel.h b/llamafile/kernel.h
index 06777161..5bdca973 100644
--- a/llamafile/kernel.h
+++ b/llamafile/kernel.h
@@ -4,12 +4,11 @@
int ytiles = (m - m0) / RM; \
int xtiles = (n - n0) / RN; \
int tiles = ytiles * xtiles; \
- double duty = (double)tiles / nth; \
+ int duty = (tiles + nth - 1) / nth; \
if (duty < 1) \
duty = 1; \
- double spot = duty * ith + .5; \
- int end = spot + duty; \
- int start = spot; \
+ int start = duty * ith; \
+ int end = start + duty; \
if (end > tiles) \
end = tiles; \
for (int job = start; job < end; ++job) { \
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 10b539f0..3fde8dd0 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -16,13 +16,9 @@
// limitations under the License.
#ifdef __x86_64__
-
+#include "sgemm.h"
#include "llamafile.h"
-
#include <cosmo.h>
-#include <stdio.h>
-
-#include "sgemm.h"
bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,
int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {
diff --git a/llamafile/sgemm_test.cpp b/llamafile/sgemm_test.cpp
index 03978f51..e633f725 100644
--- a/llamafile/sgemm_test.cpp
+++ b/llamafile/sgemm_test.cpp
@@ -36,22 +36,23 @@ float numba(void) { // (-1,1)
return float01(rand32()) * 2 - 1;
}
-template <typename T> void randomize(T *A, int n) {
- for (int i = 0; i < n; ++i)
- A[i] = numba();
-}
-
template <typename T> void broadcast(T *A, int n, T x) {
for (int i = 0; i < n; ++i)
A[i] = x;
}
+template <typename T> void randomize(int m, int n, T *A, int lda) {
+ for (int j = 0; j < n; ++j)
+ for (int i = 0; i < m; ++i)
+ A[lda * j + i] = numba();
+}
+
int main(int argc, char *argv[]) {
float tolerance = 1e-5;
- int n = 32;
- int m = 32;
- int k = 1024;
- int l = 0;
+ int m = 510;
+ int n = 513;
+ int k = 512;
+ int l = 3;
int lda = k + l;
int ldb = k + l;
int ldc = m + l;
@@ -59,11 +60,13 @@ int main(int argc, char *argv[]) {
float *B = new float[ldb * n];
float *C = new float[ldc * n];
float *G = new float[ldc * n];
- randomize(A, lda * m);
- randomize(B, ldb * n);
+ broadcast(A, lda * m, NAN);
+ broadcast(B, ldb * n, NAN);
broadcast(C, ldc * n, NAN);
broadcast(G, ldc * n, NAN);
- gemm<double>(true, false, m, n, k, 1.f, A, lda, B, ldb, 0.f, G, ldc);
+ randomize(k, m, A, lda);
+ randomize(k, n, B, ldb);
+ gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc);
if (!llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32))
return 1;