Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Revert recent upstream developments on server
Justine Tunney8ff1283d7b35
diff --git a/llama.cpp/server/httplib.h b/llama.cpp/server/httplib.hindex f360bd93..28746000 100644--- a/llama.cpp/server/httplib.h+++ b/llama.cpp/server/httplib.h@@ -1,14 +1,14 @@//// httplib.h//-// Copyright (c) 2024 Yuji Hirose. All rights reserved.+// Copyright (c) 2023 Yuji Hirose. All rights reserved.// MIT License//#ifndef CPPHTTPLIB_HTTPLIB_H#define CPPHTTPLIB_HTTPLIB_H-#define CPPHTTPLIB_VERSION "0.15.3"+#define CPPHTTPLIB_VERSION "0.12.2"/** Configuration@@ -82,10 +82,6 @@#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192#endif-#ifndef CPPHTTPLIB_RANGE_MAX_COUNT-#define CPPHTTPLIB_RANGE_MAX_COUNT 1024-#endif-#ifndef CPPHTTPLIB_TCP_NODELAY#define CPPHTTPLIB_TCP_NODELAY false#endif@@ -164,6 +160,10 @@ using ssize_t = long;#define WSA_FLAG_NO_HANDLE_INHERIT 0x80#endif+#ifndef strcasecmp+#define strcasecmp _stricmp+#endif // strcasecmp+using socket_t = SOCKET;#ifdef CPPHTTPLIB_USE_POLL#define poll(fds, nfds, timeout) WSAPoll(fds, nfds, timeout)@@ -172,15 +172,9 @@ using socket_t = SOCKET;#else // not _WIN32#include <arpa/inet.h>-#if !defined(_AIX) && !defined(__MVS__)+#ifndef _AIX#include <ifaddrs.h>#endif-#ifdef __MVS__-#include <strings.h>-#ifndef NI_MAXHOST-#define NI_MAXHOST 1025-#endif-#endif#include <net/if.h>#include <netdb.h>#include <netinet/in.h>@@ -193,7 +187,6 @@ using socket_t = SOCKET;#endif#include <csignal>#include <pthread.h>-#include <sys/mman.h>#include <sys/select.h>#include <sys/socket.h>#include <sys/un.h>@@ -214,7 +207,6 @@ using socket_t = int;#include <condition_variable>#include <cstring>#include <errno.h>-#include <exception>#include <fcntl.h>#include <fstream>#include <functional>@@ -231,9 +223,6 @@ using socket_t = int;#include <string>#include <sys/stat.h>#include <thread>-#include <unordered_map>-#include <unordered_set>-#include <utility>#ifdef CPPHTTPLIB_OPENSSL_SUPPORT#ifdef _WIN32@@ -248,6 +237,7 @@ using socket_t = int;#ifdef _MSC_VER#pragma comment(lib, "crypt32.lib")+#pragma comment(lib, "cryptui.lib")#endif#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)#include <TargetConditionals.h>@@ -269,8 +259,10 @@ using socket_t = int;#include <iostream>#include <sstream>-#if OPENSSL_VERSION_NUMBER < 0x30000000L-#error Sorry, OpenSSL versions prior to 3.0.0 are not supported+#if OPENSSL_VERSION_NUMBER < 0x1010100fL+#error Sorry, OpenSSL versions prior to 1.1.1 are not supported+#elif OPENSSL_VERSION_NUMBER < 0x30000000L+#define SSL_get1_peer_certificate SSL_get_peer_certificate#endif#endif@@ -329,7 +321,7 @@ struct scope_exit {explicit scope_exit(std::function<void(void)> &&f): exit_function(std::move(f)), execute_on_destruction{true} {}- scope_exit(scope_exit &&rhs) noexcept+ scope_exit(scope_exit &&rhs): exit_function(std::move(rhs.exit_function)),execute_on_destruction{rhs.execute_on_destruction} {rhs.release();@@ -352,81 +344,6 @@ private:} // namespace detail-enum StatusCode {- // Information responses- Continue_100 = 100,- SwitchingProtocol_101 = 101,- Processing_102 = 102,- EarlyHints_103 = 103,-- // Successful responses- OK_200 = 200,- Created_201 = 201,- Accepted_202 = 202,- NonAuthoritativeInformation_203 = 203,- NoContent_204 = 204,- ResetContent_205 = 205,- PartialContent_206 = 206,- MultiStatus_207 = 207,- AlreadyReported_208 = 208,- IMUsed_226 = 226,-- // Redirection messages- MultipleChoices_300 = 300,- MovedPermanently_301 = 301,- Found_302 = 302,- SeeOther_303 = 303,- NotModified_304 = 304,- UseProxy_305 = 305,- unused_306 = 306,- TemporaryRedirect_307 = 307,- PermanentRedirect_308 = 308,-- // Client error responses- BadRequest_400 = 400,- Unauthorized_401 = 401,- PaymentRequired_402 = 402,- Forbidden_403 = 403,- NotFound_404 = 404,- MethodNotAllowed_405 = 405,- NotAcceptable_406 = 406,- ProxyAuthenticationRequired_407 = 407,- RequestTimeout_408 = 408,- Conflict_409 = 409,- Gone_410 = 410,- LengthRequired_411 = 411,- PreconditionFailed_412 = 412,- PayloadTooLarge_413 = 413,- UriTooLong_414 = 414,- UnsupportedMediaType_415 = 415,- RangeNotSatisfiable_416 = 416,- ExpectationFailed_417 = 417,- ImATeapot_418 = 418,- MisdirectedRequest_421 = 421,- UnprocessableContent_422 = 422,- Locked_423 = 423,- FailedDependency_424 = 424,- TooEarly_425 = 425,- UpgradeRequired_426 = 426,- PreconditionRequired_428 = 428,- TooManyRequests_429 = 429,- RequestHeaderFieldsTooLarge_431 = 431,- UnavailableForLegalReasons_451 = 451,-- // Server error responses- InternalServerError_500 = 500,- NotImplemented_501 = 501,- BadGateway_502 = 502,- ServiceUnavailable_503 = 503,- GatewayTimeout_504 = 504,- HttpVersionNotSupported_505 = 505,- VariantAlsoNegotiates_506 = 506,- InsufficientStorage_507 = 507,- LoopDetected_508 = 508,- NotExtended_510 = 510,- NetworkAuthenticationRequired_511 = 511,-};-using Headers = std::multimap<std::string, std::string, detail::ci>;using Params = std::multimap<std::string, std::string>;@@ -456,18 +373,17 @@ public:DataSink &operator=(DataSink &&) = delete;std::function<bool(const char *data, size_t data_len)> write;- std::function<bool()> is_writable;std::function<void()> done;std::function<void(const Headers &trailer)> done_with_trailer;std::ostream os;private:- class data_sink_streambuf final : public std::streambuf {+ class data_sink_streambuf : public std::streambuf {public:explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {}protected:- std::streamsize xsputn(const char *s, std::streamsize n) override {+ std::streamsize xsputn(const char *s, std::streamsize n) {sink_.write(s, static_cast<size_t>(n));return n;}@@ -549,7 +465,6 @@ struct Request {MultipartFormDataMap files;Ranges ranges;Match matches;- std::unordered_map<std::string, std::string> path_params;// for clientResponseHandler response_handler;@@ -561,7 +476,8 @@ struct Request {bool has_header(const std::string &key) const;std::string get_header_value(const std::string &key, size_t id = 0) const;- uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const;+ template <typename T>+ T get_header_value(const std::string &key, size_t id = 0) const;size_t get_header_value_count(const std::string &key) const;void set_header(const std::string &key, const std::string &val);@@ -593,14 +509,14 @@ struct Response {bool has_header(const std::string &key) const;std::string get_header_value(const std::string &key, size_t id = 0) const;- uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const;+ template <typename T>+ T get_header_value(const std::string &key, size_t id = 0) const;size_t get_header_value_count(const std::string &key) const;void set_header(const std::string &key, const std::string &val);- void set_redirect(const std::string &url, int status = StatusCode::Found_302);+ void set_redirect(const std::string &url, int status = 302);void set_content(const char *s, size_t n, const std::string &content_type);void set_content(const std::string &s, const std::string &content_type);- void set_content(std::string &&s, const std::string &content_type);void set_content_provider(size_t length, const std::string &content_type, ContentProvider provider,@@ -657,16 +573,15 @@ public:TaskQueue() = default;virtual ~TaskQueue() = default;- virtual bool enqueue(std::function<void()> fn) = 0;+ virtual void enqueue(std::function<void()> fn) = 0;virtual void shutdown() = 0;virtual void on_idle() {}};-class ThreadPool final : public TaskQueue {+class ThreadPool : public TaskQueue {public:- explicit ThreadPool(size_t n, size_t mqr = 0)- : shutdown_(false), max_queued_requests_(mqr) {+ explicit ThreadPool(size_t n) : shutdown_(false) {while (n) {threads_.emplace_back(worker(*this));n--;@@ -676,17 +591,13 @@ public:ThreadPool(const ThreadPool &) = delete;~ThreadPool() override = default;- bool enqueue(std::function<void()> fn) override {+ void enqueue(std::function<void()> fn) override {{std::unique_lock<std::mutex> lock(mutex_);- if (max_queued_requests_ > 0 && jobs_.size() >= max_queued_requests_) {- return false;- }jobs_.push_back(std::move(fn));}cond_.notify_one();- return true;}void shutdown() override {@@ -736,7 +647,6 @@ private:std::list<std::function<void()>> jobs_;bool shutdown_;- size_t max_queued_requests_ = 0;std::condition_variable cond_;std::mutex mutex_;@@ -748,82 +658,6 @@ using SocketOptions = std::function<void(socket_t sock)>;void default_socket_options(socket_t sock);-const char *status_message(int status);--std::string get_bearer_token_auth(const Request &req);--namespace detail {--class MatcherBase {-public:- virtual ~MatcherBase() = default;-- // Match request path and populate its matches and- virtual bool match(Request &request) const = 0;-};--/**- * Captures parameters in request path and stores them in Request::path_params- *- * Capture name is a substring of a pattern from : to /.- * The rest of the pattern is matched agains the request path directly- * Parameters are captured starting from the next character after- * the end of the last matched static pattern fragment until the next /.- *- * Example pattern:- * "/path/fragments/:capture/more/fragments/:second_capture"- * Static fragments:- * "/path/fragments/", "more/fragments/"- *- * Given the following request path:- * "/path/fragments/:1/more/fragments/:2"- * the resulting capture will be- * {{"capture", "1"}, {"second_capture", "2"}}- */-class PathParamsMatcher final : public MatcherBase {-public:- PathParamsMatcher(const std::string &pattern);-- bool match(Request &request) const override;--private:- static constexpr char marker = ':';- // Treat segment separators as the end of path parameter capture- // Does not need to handle query parameters as they are parsed before path- // matching- static constexpr char separator = '/';-- // Contains static path fragments to match against, excluding the '/' after- // path params- // Fragments are separated by path params- std::vector<std::string> static_fragments_;- // Stores the names of the path parameters to be used as keys in the- // Request::path_params map- std::vector<std::string> param_names_;-};--/**- * Performs std::regex_match on request path- * and stores the result in Request::matches- *- * Note that regex match is performed directly on the whole request.- * This means that wildcard patterns may match multiple path segments with /:- * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end".- */-class RegexMatcher final : public MatcherBase {-public:- RegexMatcher(const std::string &pattern) : regex_(pattern) {}-- bool match(Request &request) const override;--private:- std::regex regex_;-};--ssize_t write_headers(Stream &strm, const Headers &headers);--} // namespace detail-class Server {public:using Handler = std::function<void(const Request &, Response &)>;@@ -868,7 +702,6 @@ public:bool remove_mount_point(const std::string &mount_point);Server &set_file_extension_and_mimetype_mapping(const std::string &ext,const std::string &mime);- Server &set_default_file_mimetype(const std::string &mime);Server &set_file_request_handler(Handler handler);Server &set_error_handler(HandlerWithResponse handler);@@ -885,8 +718,6 @@ public:Server &set_socket_options(SocketOptions socket_options);Server &set_default_headers(Headers headers);- Server &- set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);Server &set_keep_alive_max_count(size_t count);Server &set_keep_alive_timeout(time_t sec);@@ -934,14 +765,9 @@ protected:size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH;private:- using Handlers =- std::vector<std::pair<std::unique_ptr<detail::MatcherBase>, Handler>>;+ using Handlers = std::vector<std::pair<std::regex, Handler>>;using HandlersForContentReader =- std::vector<std::pair<std::unique_ptr<detail::MatcherBase>,- HandlerWithContentReader>>;-- static std::unique_ptr<detail::MatcherBase>- make_matcher(const std::string &pattern);+ std::vector<std::pair<std::regex, HandlerWithContentReader>>;socket_t create_server_socket(const std::string &host, int port,int socket_flags,@@ -952,16 +778,16 @@ private:bool routing(Request &req, Response &res, Stream &strm);bool handle_file_request(const Request &req, Response &res,bool head = false);- bool dispatch_request(Request &req, Response &res,- const Handlers &handlers) const;- bool dispatch_request_for_content_reader(- Request &req, Response &res, ContentReader content_reader,- const HandlersForContentReader &handlers) const;+ bool dispatch_request(Request &req, Response &res, const Handlers &handlers);+ bool+ dispatch_request_for_content_reader(Request &req, Response &res,+ ContentReader content_reader,+ const HandlersForContentReader &handlers);- bool parse_request_line(const char *s, Request &req) const;+ bool parse_request_line(const char *s, Request &req);void apply_ranges(const Request &req, Response &res,- std::string &content_type, std::string &boundary) const;- bool write_response(Stream &strm, bool close_connection, Request &req,+ std::string &content_type, std::string &boundary);+ bool write_response(Stream &strm, bool close_connection, const Request &req,Response &res);bool write_response_with_content(Stream &strm, bool close_connection,const Request &req, Response &res);@@ -980,23 +806,21 @@ private:bool read_content_core(Stream &strm, Request &req, Response &res,ContentReceiver receiver,MultipartContentHeader multipart_header,- ContentReceiver multipart_receiver) const;+ ContentReceiver multipart_receiver);virtual bool process_and_close_socket(socket_t sock);- std::atomic<bool> is_running_{false};- std::atomic<bool> done_{false};-struct MountPointEntry {std::string mount_point;std::string base_dir;Headers headers;};std::vector<MountPointEntry> base_dirs_;++ std::atomic<bool> is_running_{false};+ std::atomic<bool> done_{false};std::map<std::string, std::string> file_extension_and_mimetype_map_;- std::string default_file_mimetype_ = "application/octet-stream";Handler file_request_handler_;-Handlers get_handlers_;Handlers post_handlers_;HandlersForContentReader post_handlers_for_content_reader_;@@ -1007,22 +831,18 @@ private:Handlers delete_handlers_;HandlersForContentReader delete_handlers_for_content_reader_;Handlers options_handlers_;-HandlerWithResponse error_handler_;ExceptionHandler exception_handler_;HandlerWithResponse pre_routing_handler_;Handler post_routing_handler_;- Expect100ContinueHandler expect_100_continue_handler_;-Logger logger_;+ Expect100ContinueHandler expect_100_continue_handler_;int address_family_ = AF_UNSPEC;bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY;SocketOptions socket_options_ = default_socket_options;Headers default_headers_;- std::function<ssize_t(Stream &, Headers &)> header_writer_ =- detail::write_headers;};enum class Error {@@ -1040,19 +860,17 @@ enum class Error {UnsupportedMultipartBoundaryChars,Compression,ConnectionTimeout,- ProxyConnection,// For internal use onlySSLPeerCouldBeClosed_,};-std::string to_string(Error error);+std::string to_string(const Error error);std::ostream &operator<<(std::ostream &os, const Error &obj);class Result {public:- Result() = default;Result(std::unique_ptr<Response> &&res, Error err,Headers &&request_headers = Headers{}): res_(std::move(res)), err_(err),@@ -1075,13 +893,13 @@ public:bool has_request_header(const std::string &key) const;std::string get_request_header_value(const std::string &key,size_t id = 0) const;- uint64_t get_request_header_value_u64(const std::string &key,- size_t id = 0) const;+ template <typename T>+ T get_request_header_value(const std::string &key, size_t id = 0) const;size_t get_request_header_value_count(const std::string &key) const;private:std::unique_ptr<Response> res_;- Error err_ = Error::Unknown;+ Error err_;Headers request_headers_;};@@ -1241,21 +1059,16 @@ public:bool send(Request &req, Response &res, Error &error);Result send(const Request &req);- void stop();-- std::string host() const;- int port() const;-size_t is_socket_open() const;+socket_t socket() const;+ void stop();+void set_hostname_addr_map(std::map<std::string, std::string> addr_map);void set_default_headers(Headers headers);- void- set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);-void set_address_family(int family);void set_tcp_nodelay(bool on);void set_socket_options(SocketOptions socket_options);@@ -1304,7 +1117,6 @@ public:void set_ca_cert_path(const std::string &ca_cert_file_path,const std::string &ca_cert_dir_path = std::string());void set_ca_cert_store(X509_STORE *ca_cert_store);- X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size) const;#endif#ifdef CPPHTTPLIB_OPENSSL_SUPPORT@@ -1333,14 +1145,14 @@ protected:// Also, shutdown_ssl and close_socket should also NOT be called concurrently// with a DIFFERENT thread sending requests using that socket.virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully);- void shutdown_socket(Socket &socket) const;+ void shutdown_socket(Socket &socket);void close_socket(Socket &socket);bool process_request(Stream &strm, Request &req, Response &res,bool close_connection, Error &error);bool write_content_with_provider(Stream &strm, const Request &req,- Error &error) const;+ Error &error);void copy_settings(const ClientImpl &rhs);@@ -1365,10 +1177,6 @@ protected:// Default headersHeaders default_headers_;- // Header writer- std::function<ssize_t(Stream &, Headers &)> header_writer_ =- detail::write_headers;-// Settingsstd::string client_cert_path_;std::string client_key_path_;@@ -1431,8 +1239,7 @@ private:Result send_(Request &&req);socket_t create_client_socket(Error &error) const;- bool read_response_line(Stream &strm, const Request &req,- Response &res) const;+ bool read_response_line(Stream &strm, const Request &req, Response &res);bool write_request(Stream &strm, Request &req, bool close_connection,Error &error);bool redirect(Request &req, Response &res, Error &error);@@ -1451,7 +1258,7 @@ private:const std::string &content_type);ContentProviderWithoutLength get_multipart_content_provider(const std::string &boundary, const MultipartFormDataItems &items,- const MultipartFormDataProviderItems &provider_items) const;+ const MultipartFormDataProviderItems &provider_items);std::string adjust_host_string(const std::string &host) const;@@ -1624,21 +1431,16 @@ public:bool send(Request &req, Response &res, Error &error);Result send(const Request &req);- void stop();-- std::string host() const;- int port() const;-size_t is_socket_open() const;+socket_t socket() const;+ void stop();+void set_hostname_addr_map(std::map<std::string, std::string> addr_map);void set_default_headers(Headers headers);- void- set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);-void set_address_family(int family);void set_tcp_nodelay(bool on);void set_socket_options(SocketOptions socket_options);@@ -1695,7 +1497,6 @@ public:const std::string &ca_cert_dir_path = std::string());void set_ca_cert_store(X509_STORE *ca_cert_store);- void load_ca_cert_store(const char *ca_cert, std::size_t size);long get_openssl_verify_result() const;@@ -1737,7 +1538,7 @@ private:std::mutex ctx_mutex_;};-class SSLClient final : public ClientImpl {+class SSLClient : public ClientImpl {public:explicit SSLClient(const std::string &host);@@ -1745,19 +1546,16 @@ public:explicit SSLClient(const std::string &host, int port,const std::string &client_cert_path,- const std::string &client_key_path,- const std::string &private_key_password = std::string());+ const std::string &client_key_path);explicit SSLClient(const std::string &host, int port, X509 *client_cert,- EVP_PKEY *client_key,- const std::string &private_key_password = std::string());+ EVP_PKEY *client_key);~SSLClient() override;bool is_valid() const override;void set_ca_cert_store(X509_STORE *ca_cert_store);- void load_ca_cert_store(const char *ca_cert, std::size_t size);long get_openssl_verify_result() const;@@ -1766,7 +1564,7 @@ public:private:bool create_and_connect_socket(Socket &socket, Error &error) override;void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override;- void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully);+ void shutdown_ssl_impl(Socket &socket, bool shutdown_socket);bool process_socket(const Socket &socket,std::function<bool(Stream &strm)> callback) override;@@ -1810,9 +1608,15 @@ inline void duration_to_sec_and_usec(const T &duration, U callback) {callback(static_cast<time_t>(sec), static_cast<time_t>(usec));}-inline uint64_t get_header_value_u64(const Headers &headers,- const std::string &key, size_t id,- uint64_t def) {+template <typename T>+inline T get_header_value(const Headers & /*headers*/,+ const std::string & /*key*/, size_t /*id*/ = 0,+ uint64_t /*def*/ = 0) {}++template <>+inline uint64_t get_header_value<uint64_t>(const Headers &headers,+ const std::string &key, size_t id,+ uint64_t def) {auto rng = headers.equal_range(key);auto it = rng.first;std::advance(it, static_cast<ssize_t>(id));@@ -1824,14 +1628,14 @@ inline uint64_t get_header_value_u64(const Headers &headers,} // namespace detail-inline uint64_t Request::get_header_value_u64(const std::string &key,- size_t id) const {- return detail::get_header_value_u64(headers, key, id, 0);+template <typename T>+inline T Request::get_header_value(const std::string &key, size_t id) const {+ return detail::get_header_value<T>(headers, key, id, 0);}-inline uint64_t Response::get_header_value_u64(const std::string &key,- size_t id) const {- return detail::get_header_value_u64(headers, key, id, 0);+template <typename T>+inline T Response::get_header_value(const std::string &key, size_t id) const {+ return detail::get_header_value<T>(headers, key, id, 0);}template <typename... Args>@@ -1861,106 +1665,21 @@ inline ssize_t Stream::write_format(const char *fmt, const Args &...args) {inline void default_socket_options(socket_t sock) {int yes = 1;#ifdef _WIN32- setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,- reinterpret_cast<const char *>(&yes), sizeof(yes));+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char *>(&yes),+ sizeof(yes));setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,- reinterpret_cast<const char *>(&yes), sizeof(yes));+ reinterpret_cast<char *>(&yes), sizeof(yes));#else#ifdef SO_REUSEPORT- setsockopt(sock, SOL_SOCKET, SO_REUSEPORT,- reinterpret_cast<const void *>(&yes), sizeof(yes));+ setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast<void *>(&yes),+ sizeof(yes));#else- setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,- reinterpret_cast<const void *>(&yes), sizeof(yes));+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<void *>(&yes),+ sizeof(yes));#endif#endif}-inline const char *status_message(int status) {- switch (status) {- case StatusCode::Continue_100: return "Continue";- case StatusCode::SwitchingProtocol_101: return "Switching Protocol";- case StatusCode::Processing_102: return "Processing";- case StatusCode::EarlyHints_103: return "Early Hints";- case StatusCode::OK_200: return "OK";- case StatusCode::Created_201: return "Created";- case StatusCode::Accepted_202: return "Accepted";- case StatusCode::NonAuthoritativeInformation_203:- return "Non-Authoritative Information";- case StatusCode::NoContent_204: return "No Content";- case StatusCode::ResetContent_205: return "Reset Content";- case StatusCode::PartialContent_206: return "Partial Content";- case StatusCode::MultiStatus_207: return "Multi-Status";- case StatusCode::AlreadyReported_208: return "Already Reported";- case StatusCode::IMUsed_226: return "IM Used";- case StatusCode::MultipleChoices_300: return "Multiple Choices";- case StatusCode::MovedPermanently_301: return "Moved Permanently";- case StatusCode::Found_302: return "Found";- case StatusCode::SeeOther_303: return "See Other";- case StatusCode::NotModified_304: return "Not Modified";- case StatusCode::UseProxy_305: return "Use Proxy";- case StatusCode::unused_306: return "unused";- case StatusCode::TemporaryRedirect_307: return "Temporary Redirect";- case StatusCode::PermanentRedirect_308: return "Permanent Redirect";- case StatusCode::BadRequest_400: return "Bad Request";- case StatusCode::Unauthorized_401: return "Unauthorized";- case StatusCode::PaymentRequired_402: return "Payment Required";- case StatusCode::Forbidden_403: return "Forbidden";- case StatusCode::NotFound_404: return "Not Found";- case StatusCode::MethodNotAllowed_405: return "Method Not Allowed";- case StatusCode::NotAcceptable_406: return "Not Acceptable";- case StatusCode::ProxyAuthenticationRequired_407:- return "Proxy Authentication Required";- case StatusCode::RequestTimeout_408: return "Request Timeout";- case StatusCode::Conflict_409: return "Conflict";- case StatusCode::Gone_410: return "Gone";- case StatusCode::LengthRequired_411: return "Length Required";- case StatusCode::PreconditionFailed_412: return "Precondition Failed";- case StatusCode::PayloadTooLarge_413: return "Payload Too Large";- case StatusCode::UriTooLong_414: return "URI Too Long";- case StatusCode::UnsupportedMediaType_415: return "Unsupported Media Type";- case StatusCode::RangeNotSatisfiable_416: return "Range Not Satisfiable";- case StatusCode::ExpectationFailed_417: return "Expectation Failed";- case StatusCode::ImATeapot_418: return "I'm a teapot";- case StatusCode::MisdirectedRequest_421: return "Misdirected Request";- case StatusCode::UnprocessableContent_422: return "Unprocessable Content";- case StatusCode::Locked_423: return "Locked";- case StatusCode::FailedDependency_424: return "Failed Dependency";- case StatusCode::TooEarly_425: return "Too Early";- case StatusCode::UpgradeRequired_426: return "Upgrade Required";- case StatusCode::PreconditionRequired_428: return "Precondition Required";- case StatusCode::TooManyRequests_429: return "Too Many Requests";- case StatusCode::RequestHeaderFieldsTooLarge_431:- return "Request Header Fields Too Large";- case StatusCode::UnavailableForLegalReasons_451:- return "Unavailable For Legal Reasons";- case StatusCode::NotImplemented_501: return "Not Implemented";- case StatusCode::BadGateway_502: return "Bad Gateway";- case StatusCode::ServiceUnavailable_503: return "Service Unavailable";- case StatusCode::GatewayTimeout_504: return "Gateway Timeout";- case StatusCode::HttpVersionNotSupported_505:- return "HTTP Version Not Supported";- case StatusCode::VariantAlsoNegotiates_506: return "Variant Also Negotiates";- case StatusCode::InsufficientStorage_507: return "Insufficient Storage";- case StatusCode::LoopDetected_508: return "Loop Detected";- case StatusCode::NotExtended_510: return "Not Extended";- case StatusCode::NetworkAuthenticationRequired_511:- return "Network Authentication Required";-- default:- case StatusCode::InternalServerError_500: return "Internal Server Error";- }-}--inline std::string get_bearer_token_auth(const Request &req) {- if (req.has_header("Authorization")) {- static std::string BearerHeaderPrefix = "Bearer ";- return req.get_header_value("Authorization")- .substr(BearerHeaderPrefix.length());- }- return "";-}-template <class Rep, class Period>inline Server &Server::set_read_timeout(const std::chrono::duration<Rep, Period> &duration) {@@ -2001,7 +1720,6 @@ inline std::string to_string(const Error error) {return "Unsupported HTTP multipart boundary characters";case Error::Compression: return "Compression failed";case Error::ConnectionTimeout: return "Connection timed out";- case Error::ProxyConnection: return "Proxy connection failed";case Error::Unknown: return "Unknown";default: break;}@@ -2015,9 +1733,10 @@ inline std::ostream &operator<<(std::ostream &os, const Error &obj) {return os;}-inline uint64_t Result::get_request_header_value_u64(const std::string &key,- size_t id) const {- return detail::get_header_value_u64(request_headers_, key, id, 0);+template <typename T>+inline T Result::get_request_header_value(const std::string &key,+ size_t id) const {+ return detail::get_header_value<T>(request_headers_, key, id, 0);}template <class Rep, class Period>@@ -2071,7 +1790,7 @@ void hosted_at(const std::string &hostname, std::vector<std::string> &addrs);std::string append_query_params(const std::string &path, const Params ¶ms);-std::pair<std::string, std::string> make_range_header(const Ranges &ranges);+std::pair<std::string, std::string> make_range_header(Ranges ranges);std::pair<std::string, std::string>make_basic_authentication_header(const std::string &username,@@ -2091,9 +1810,6 @@ std::string trim_copy(const std::string &s);void split(const char *b, const char *e, char d,std::function<void(const char *, const char *)> fn);-void split(const char *b, const char *e, char d, size_t m,- std::function<void(const char *, const char *)> fn);-bool process_client_socket(socket_t sock, time_t read_timeout_sec,time_t read_timeout_usec, time_t write_timeout_sec,time_t write_timeout_usec,@@ -2128,7 +1844,7 @@ enum class EncodingType { None = 0, Gzip, Brotli };EncodingType encoding_type(const Request &req, const Response &res);-class BufferStream final : public Stream {+class BufferStream : public Stream {public:BufferStream() = default;~BufferStream() override = default;@@ -2168,19 +1884,19 @@ public:Callback callback) = 0;};-class nocompressor final : public compressor {+class nocompressor : public compressor {public:- ~nocompressor() override = default;+ virtual ~nocompressor() = default;bool compress(const char *data, size_t data_length, bool /*last*/,Callback callback) override;};#ifdef CPPHTTPLIB_ZLIB_SUPPORT-class gzip_compressor final : public compressor {+class gzip_compressor : public compressor {public:gzip_compressor();- ~gzip_compressor() override;+ ~gzip_compressor();bool compress(const char *data, size_t data_length, bool last,Callback callback) override;@@ -2190,10 +1906,10 @@ private:z_stream strm_;};-class gzip_decompressor final : public decompressor {+class gzip_decompressor : public decompressor {public:gzip_decompressor();- ~gzip_decompressor() override;+ ~gzip_decompressor();bool is_valid() const override;@@ -2207,7 +1923,7 @@ private:#endif#ifdef CPPHTTPLIB_BROTLI_SUPPORT-class brotli_compressor final : public compressor {+class brotli_compressor : public compressor {public:brotli_compressor();~brotli_compressor();@@ -2219,7 +1935,7 @@ private:BrotliEncoderState *state_ = nullptr;};-class brotli_decompressor final : public decompressor {+class brotli_decompressor : public decompressor {public:brotli_decompressor();~brotli_decompressor();@@ -2256,29 +1972,6 @@ private:std::string glowable_buffer_;};-class mmap {-public:- mmap(const char *path);- ~mmap();-- bool open(const char *path);- void close();-- bool is_open() const;- size_t size() const;- const char *data() const;--private:-#if defined(_WIN32)- HANDLE hFile_;- HANDLE hMapping_;-#else- int fd_;-#endif- size_t size_;- void *addr_;-};-} // namespace detail// ----------------------------------------------------------------------------@@ -2310,7 +2003,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,val = 0;for (; cnt; i++, cnt--) {if (!s[i]) { return false; }- auto v = 0;+ int v = 0;if (is_hex(s[i], v)) {val = val * 16 + v;} else {@@ -2321,7 +2014,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,}inline std::string from_i_to_hex(size_t n) {- static const auto charset = "0123456789abcdef";+ const char *charset = "0123456789abcdef";std::string ret;do {ret = charset[n & 15] + ret;@@ -2332,7 +2025,7 @@ inline std::string from_i_to_hex(size_t n) {inline size_t to_utf8(int code, char *buff) {if (code < 0x0080) {- buff[0] = static_cast<char>(code & 0x7F);+ buff[0] = (code & 0x7F);return 1;} else if (code < 0x0800) {buff[0] = static_cast<char>(0xC0 | ((code >> 6) & 0x1F));@@ -2371,8 +2064,8 @@ inline std::string base64_encode(const std::string &in) {std::string out;out.reserve(in.size());- auto val = 0;- auto valb = -6;+ int val = 0;+ int valb = -6;for (auto c : in) {val = (val << 8) + static_cast<uint8_t>(c);@@ -2419,11 +2112,6 @@ inline bool is_valid_path(const std::string &path) {// Read componentauto beg = i;while (i < path.size() && path[i] != '/') {- if (path[i] == '\0') {- return false;- } else if (path[i] == '\\') {- return false;- }i++;}@@ -2508,7 +2196,7 @@ inline std::string decode_url(const std::string &s,for (size_t i = 0; i < s.size(); i++) {if (s[i] == '%' && i + 1 < s.size()) {if (s[i + 1] == 'u') {- auto val = 0;+ int val = 0;if (from_hex_to_i(s, i + 2, 4, val)) {// 4 digits Unicode codeschar buff[4];@@ -2519,7 +2207,7 @@ inline std::string decode_url(const std::string &s,result += s[i];}} else {- auto val = 0;+ int val = 0;if (from_hex_to_i(s, i + 1, 2, val)) {// 2 digits hex codesresult += static_cast<char>(val);@@ -2572,30 +2260,16 @@ inline std::string trim_copy(const std::string &s) {return s.substr(r.first, r.second - r.first);}-inline std::string trim_double_quotes_copy(const std::string &s) {- if (s.length() >= 2 && s.front() == '"' && s.back() == '"') {- return s.substr(1, s.size() - 2);- }- return s;-}-inline void split(const char *b, const char *e, char d,std::function<void(const char *, const char *)> fn) {- return split(b, e, d, (std::numeric_limits<size_t>::max)(), std::move(fn));-}--inline void split(const char *b, const char *e, char d, size_t m,- std::function<void(const char *, const char *)> fn) {size_t i = 0;size_t beg = 0;- size_t count = 1;while (e ? (b + i < e) : (b[i] != '\0')) {- if (b[i] == d && count < m) {+ if (b[i] == d) {auto r = trim(b, e, beg, i);if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }beg = i + 1;- count++;}i++;}@@ -2671,105 +2345,6 @@ inline void stream_line_reader::append(char c) {}}-inline mmap::mmap(const char *path)-#if defined(_WIN32)- : hFile_(NULL), hMapping_(NULL)-#else- : fd_(-1)-#endif- ,- size_(0), addr_(nullptr) {- open(path);-}--inline mmap::~mmap() { close(); }--inline bool mmap::open(const char *path) {- close();--#if defined(_WIN32)- std::wstring wpath;- for (size_t i = 0; i < strlen(path); i++) {- wpath += path[i];- }-- hFile_ = ::CreateFile2(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ,- OPEN_EXISTING, NULL);-- if (hFile_ == INVALID_HANDLE_VALUE) { return false; }-- LARGE_INTEGER size{};- if (!::GetFileSizeEx(hFile_, &size)) { return false; }- size_ = static_cast<size_t>(size.QuadPart);-- hMapping_ =- ::CreateFileMappingFromApp(hFile_, NULL, PAGE_READONLY, size_, NULL);-- if (hMapping_ == NULL) {- close();- return false;- }-- addr_ = ::MapViewOfFileFromApp(hMapping_, FILE_MAP_READ, 0, 0);-#else- fd_ = ::open(path, O_RDONLY);- if (fd_ == -1) { return false; }-- struct stat sb;- if (fstat(fd_, &sb) == -1) {- close();- return false;- }- size_ = static_cast<size_t>(sb.st_size);-- addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0);-#endif-- if (addr_ == nullptr) {- close();- return false;- }-- return true;-}--inline bool mmap::is_open() const { return addr_ != nullptr; }--inline size_t mmap::size() const { return size_; }--inline const char *mmap::data() const {- return static_cast<const char *>(addr_);-}--inline void mmap::close() {-#if defined(_WIN32)- if (addr_) {- ::UnmapViewOfFile(addr_);- addr_ = nullptr;- }-- if (hMapping_) {- ::CloseHandle(hMapping_);- hMapping_ = NULL;- }-- if (hFile_ != INVALID_HANDLE_VALUE) {- ::CloseHandle(hFile_);- hFile_ = INVALID_HANDLE_VALUE;- }-#else- if (addr_ != nullptr) {- munmap(addr_, size_);- addr_ = nullptr;- }-- if (fd_ != -1) {- ::close(fd_);- fd_ = -1;- }-#endif- size_ = 0;-}inline int close_socket(socket_t sock) {#ifdef _WIN32return closesocket(sock);@@ -2779,7 +2354,7 @@ inline int close_socket(socket_t sock) {}template <typename T> inline ssize_t handle_EINTR(T fn) {- ssize_t res = 0;+ ssize_t res = false;while (true) {res = fn();if (res < 0 && errno == EINTR) { continue; }@@ -2824,7 +2399,7 @@ inline ssize_t select_read(socket_t sock, time_t sec, time_t usec) {return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });#else#ifndef _WIN32- if (sock >= FD_SETSIZE) { return -1; }+ if (sock >= FD_SETSIZE) { return 1; }#endiffd_set fds;@@ -2852,7 +2427,7 @@ inline ssize_t select_write(socket_t sock, time_t sec, time_t usec) {return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });#else#ifndef _WIN32- if (sock >= FD_SETSIZE) { return -1; }+ if (sock >= FD_SETSIZE) { return 1; }#endiffd_set fds;@@ -2883,7 +2458,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec,if (poll_res == 0) { return Error::ConnectionTimeout; }if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) {- auto error = 0;+ int error = 0;socklen_t len = sizeof(error);auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,reinterpret_cast<char *>(&error), &len);@@ -2915,7 +2490,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec,if (ret == 0) { return Error::ConnectionTimeout; }if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) {- auto error = 0;+ int error = 0;socklen_t len = sizeof(error);auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,reinterpret_cast<char *>(&error), &len);@@ -2937,7 +2512,7 @@ inline bool is_socket_alive(socket_t sock) {return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0;}-class SocketStream final : public Stream {+class SocketStream : public Stream {public:SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,time_t write_timeout_sec, time_t write_timeout_usec);@@ -2962,11 +2537,11 @@ private:size_t read_buff_off_ = 0;size_t read_buff_content_size_ = 0;- static const size_t read_buff_size_ = 1024l * 4;+ static const size_t read_buff_size_ = 1024 * 4;};#ifdef CPPHTTPLIB_OPENSSL_SUPPORT-class SSLSocketStream final : public Stream {+class SSLSocketStream : public Stream {public:SSLSocketStream(socket_t sock, SSL *ssl, time_t read_timeout_sec,time_t read_timeout_usec, time_t write_timeout_sec,@@ -3091,7 +2666,7 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,#ifndef _WIN32if (hints.ai_family == AF_UNIX) {const auto addrlen = host.length();- if (addrlen > sizeof(sockaddr_un::sun_path)) { return INVALID_SOCKET; }+ if (addrlen > sizeof(sockaddr_un::sun_path)) return INVALID_SOCKET;auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol);if (sock != INVALID_SOCKET) {@@ -3160,27 +2735,17 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,#endifif (tcp_nodelay) {- auto yes = 1;-#ifdef _WIN32- setsockopt(sock, IPPROTO_TCP, TCP_NODELAY,- reinterpret_cast<const char *>(&yes), sizeof(yes));-#else- setsockopt(sock, IPPROTO_TCP, TCP_NODELAY,- reinterpret_cast<const void *>(&yes), sizeof(yes));-#endif+ int yes = 1;+ setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char *>(&yes),+ sizeof(yes));}if (socket_options) { socket_options(sock); }if (rp->ai_family == AF_INET6) {- auto no = 0;-#ifdef _WIN32- setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY,- reinterpret_cast<const char *>(&no), sizeof(no));-#else- setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY,- reinterpret_cast<const void *>(&no), sizeof(no));-#endif+ int no = 0;+ setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast<char *>(&no),+ sizeof(no));}// bind or connect@@ -3239,7 +2804,7 @@ inline bool bind_ip_address(socket_t sock, const std::string &host) {return ret;}-#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__+#if !defined _WIN32 && !defined ANDROID && !defined _AIX#define USE_IF2IP#endif@@ -3295,7 +2860,7 @@ inline socket_t create_client_socket(#ifdef USE_IF2IPauto ip_from_if = if2ip(address_family, intf);if (ip_from_if.empty()) { ip_from_if = intf; }- if (!bind_ip_address(sock2, ip_from_if)) {+ if (!bind_ip_address(sock2, ip_from_if.c_str())) {error = Error::BindIPAddress;return false;}@@ -3323,14 +2888,13 @@ inline socket_t create_client_socket(#ifdef _WIN32auto timeout = static_cast<uint32_t>(read_timeout_sec * 1000 +read_timeout_usec / 1000);- setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO,- reinterpret_cast<const char *>(&timeout), sizeof(timeout));+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,+ sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(read_timeout_sec);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec);- setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO,- reinterpret_cast<const void *>(&tv), sizeof(tv));+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));#endif}{@@ -3338,14 +2902,13 @@ inline socket_t create_client_socket(#ifdef _WIN32auto timeout = static_cast<uint32_t>(write_timeout_sec * 1000 +write_timeout_usec / 1000);- setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO,- reinterpret_cast<const char *>(&timeout), sizeof(timeout));+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,+ sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(write_timeout_sec);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec);- setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO,- reinterpret_cast<const void *>(&tv), sizeof(tv));+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));#endif}@@ -3445,20 +3008,18 @@ inline constexpr unsigned int operator"" _t(const char *s, size_t l) {} // namespace udl-inline std::string+inline const char *find_content_type(const std::string &path,- const std::map<std::string, std::string> &user_data,- const std::string &default_content_type) {+ const std::map<std::string, std::string> &user_data) {auto ext = file_extension(path);auto it = user_data.find(ext);- if (it != user_data.end()) { return it->second; }+ if (it != user_data.end()) { return it->second.c_str(); }using udl::operator""_t;switch (str2tag(ext)) {- default: return default_content_type;-+ default: return nullptr;case "css"_t: return "text/css";case "csv"_t: return "text/csv";case "htm"_t:@@ -3511,6 +3072,76 @@ find_content_type(const std::string &path,}}+inline const char *status_message(int status) {+ switch (status) {+ case 100: return "Continue";+ case 101: return "Switching Protocol";+ case 102: return "Processing";+ case 103: return "Early Hints";+ case 200: return "OK";+ case 201: return "Created";+ case 202: return "Accepted";+ case 203: return "Non-Authoritative Information";+ case 204: return "No Content";+ case 205: return "Reset Content";+ case 206: return "Partial Content";+ case 207: return "Multi-Status";+ case 208: return "Already Reported";+ case 226: return "IM Used";+ case 300: return "Multiple Choice";+ case 301: return "Moved Permanently";+ case 302: return "Found";+ case 303: return "See Other";+ case 304: return "Not Modified";+ case 305: return "Use Proxy";+ case 306: return "unused";+ case 307: return "Temporary Redirect";+ case 308: return "Permanent Redirect";+ case 400: return "Bad Request";+ case 401: return "Unauthorized";+ case 402: return "Payment Required";+ case 403: return "Forbidden";+ case 404: return "Not Found";+ case 405: return "Method Not Allowed";+ case 406: return "Not Acceptable";+ case 407: return "Proxy Authentication Required";+ case 408: return "Request Timeout";+ case 409: return "Conflict";+ case 410: return "Gone";+ case 411: return "Length Required";+ case 412: return "Precondition Failed";+ case 413: return "Payload Too Large";+ case 414: return "URI Too Long";+ case 415: return "Unsupported Media Type";+ case 416: return "Range Not Satisfiable";+ case 417: return "Expectation Failed";+ case 418: return "I'm a teapot";+ case 421: return "Misdirected Request";+ case 422: return "Unprocessable Entity";+ case 423: return "Locked";+ case 424: return "Failed Dependency";+ case 425: return "Too Early";+ case 426: return "Upgrade Required";+ case 428: return "Precondition Required";+ case 429: return "Too Many Requests";+ case 431: return "Request Header Fields Too Large";+ case 451: return "Unavailable For Legal Reasons";+ case 501: return "Not Implemented";+ case 502: return "Bad Gateway";+ case 503: return "Service Unavailable";+ case 504: return "Gateway Timeout";+ case 505: return "HTTP Version Not Supported";+ case 506: return "Variant Also Negotiates";+ case 507: return "Insufficient Storage";+ case 508: return "Loop Detected";+ case 510: return "Not Extended";+ case 511: return "Network Authentication Required";++ default:+ case 500: return "Internal Server Error";+ }+}+inline bool can_compress_content_type(const std::string &content_type) {using udl::operator""_t;@@ -3587,7 +3218,7 @@ inline bool gzip_compressor::compress(const char *data, size_t data_length,data += strm_.avail_in;auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH;- auto ret = Z_OK;+ int ret = Z_OK;std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};do {@@ -3631,7 +3262,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,Callback callback) {assert(is_valid_);- auto ret = Z_OK;+ int ret = Z_OK;do {constexpr size_t max_avail_in =@@ -3645,12 +3276,16 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,data += strm_.avail_in;std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};- while (strm_.avail_in > 0 && ret == Z_OK) {+ while (strm_.avail_in > 0) {strm_.avail_out = static_cast<uInt>(buff.size());strm_.next_out = reinterpret_cast<Bytef *>(buff.data());+ auto prev_avail_in = strm_.avail_in;+ret = inflate(&strm_, Z_NO_FLUSH);+ if (prev_avail_in - strm_.avail_in == 0) { return false; }+assert(ret != Z_STREAM_ERROR);switch (ret) {case Z_NEED_DICT:@@ -3663,7 +3298,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,}}- if (ret != Z_OK && ret != Z_STREAM_END) { return false; }+ if (ret != Z_OK && ret != Z_STREAM_END) return false;} while (data_length > 0);@@ -3732,7 +3367,7 @@ inline bool brotli_decompressor::decompress(const char *data,return 0;}- auto next_in = reinterpret_cast<const uint8_t *>(data);+ const uint8_t *next_in = (const uint8_t *)data;size_t avail_in = data_length;size_t total_out;@@ -3802,9 +3437,6 @@ inline bool parse_header(const char *beg, const char *end, T fn) {}if (p < end) {- auto key_len = key_end - beg;- if (!key_len) { return false; }-auto key = std::string(beg, key_end);auto val = compare_case_ignore(key, "Location")? std::string(p, end)@@ -3894,7 +3526,11 @@ inline bool read_content_without_length(Stream &strm,uint64_t r = 0;for (;;) {auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ);- if (n <= 0) { return true; }+ if (n < 0) {+ return false;+ } else if (n == 0) {+ return true;+ }if (!out(buf, static_cast<size_t>(n), r, 0)) { return false; }r += static_cast<uint64_t>(n);@@ -3930,7 +3566,7 @@ inline bool read_content_chunked(Stream &strm, T &x,if (!line_reader.getline()) { return false; }- if (strcmp(line_reader.ptr(), "\r\n") != 0) { return false; }+ if (strcmp(line_reader.ptr(), "\r\n")) { return false; }if (!line_reader.getline()) { return false; }}@@ -3940,7 +3576,7 @@ inline bool read_content_chunked(Stream &strm, T &x,// Trailerif (!line_reader.getline()) { return false; }- while (strcmp(line_reader.ptr(), "\r\n") != 0) {+ while (strcmp(line_reader.ptr(), "\r\n")) {if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }// Exclude line terminator@@ -3959,8 +3595,8 @@ inline bool read_content_chunked(Stream &strm, T &x,}inline bool is_chunked_transfer_encoding(const Headers &headers) {- return compare_case_ignore(- get_header_value(headers, "Transfer-Encoding", 0, ""), "chunked");+ return !strcasecmp(get_header_value(headers, "Transfer-Encoding", 0, ""),+ "chunked");}template <typename T, typename U>@@ -3975,14 +3611,14 @@ bool prepare_content_receiver(T &x, int &status,#ifdef CPPHTTPLIB_ZLIB_SUPPORTdecompressor = detail::make_unique<gzip_decompressor>();#else- status = StatusCode::UnsupportedMediaType_415;+ status = 415;return false;#endif} else if (encoding.find("br") != std::string::npos) {#ifdef CPPHTTPLIB_BROTLI_SUPPORTdecompressor = detail::make_unique<brotli_decompressor>();#else- status = StatusCode::UnsupportedMediaType_415;+ status = 415;return false;#endif}@@ -3998,7 +3634,7 @@ bool prepare_content_receiver(T &x, int &status,};return callback(std::move(out));} else {- status = StatusCode::InternalServerError_500;+ status = 500;return false;}}@@ -4026,7 +3662,7 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,} else if (!has_header(x.headers, "Content-Length")) {ret = read_content_without_length(strm, out);} else {- auto len = get_header_value_u64(x.headers, "Content-Length", 0, 0);+ auto len = get_header_value<uint64_t>(x.headers, "Content-Length");if (len > payload_max_length) {exceed_payload_max_length = true;skip_content_with_length(strm, len);@@ -4036,10 +3672,7 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,}}- if (!ret) {- status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413- : StatusCode::BadRequest_400;- }+ if (!ret) { status = exceed_payload_max_length ? 413 : 400; }return ret;});} // namespace detail@@ -4087,8 +3720,6 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider,return ok;};- data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };-while (offset < end_offset && !is_shutting_down()) {if (!strm.is_writable()) {error = Error::Write;@@ -4133,8 +3764,6 @@ write_content_without_length(Stream &strm,return ok;};- data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };-data_sink.done = [&](void) { data_available = false; };while (data_available && !is_shutting_down()) {@@ -4185,8 +3814,6 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider,return ok;};- data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };-auto done_with_trailer = [&](const Headers *trailer) {if (!ok) { return; }@@ -4271,8 +3898,7 @@ inline bool redirect(T &cli, Request &req, Response &res,new_req.path = path;new_req.redirect_count_ -= 1;- if (res.status == StatusCode::SeeOther_303 &&- (req.method != "GET" && req.method != "HEAD")) {+ if (res.status == 303 && (req.method != "GET" && req.method != "HEAD")) {new_req.method = "GET";new_req.body.clear();new_req.headers.clear();@@ -4284,8 +3910,7 @@ inline bool redirect(T &cli, Request &req, Response &res,if (ret) {req = new_req;res = new_res;-- if (res.location.empty()) { res.location = location; }+ res.location = location;}return ret;}@@ -4332,34 +3957,14 @@ inline bool parse_multipart_boundary(const std::string &content_type,if (pos == std::string::npos) { return false; }auto end = content_type.find(';', pos);auto beg = pos + strlen(boundary_keyword);- boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg));+ boundary = content_type.substr(beg, end - beg);+ if (boundary.length() >= 2 && boundary.front() == '"' &&+ boundary.back() == '"') {+ boundary = boundary.substr(1, boundary.size() - 2);+ }return !boundary.empty();}-inline void parse_disposition_params(const std::string &s, Params ¶ms) {- std::set<std::string> cache;- split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) {- std::string kv(b, e);- if (cache.find(kv) != cache.end()) { return; }- cache.insert(kv);-- std::string key;- std::string val;- split(b, e, '=', [&](const char *b2, const char *e2) {- if (key.empty()) {- key.assign(b2, e2);- } else {- val.assign(b2, e2);- }- });-- if (!key.empty()) {- params.emplace(trim_double_quotes_copy((key)),- trim_double_quotes_copy((val)));- }- });-}-#ifdef CPPHTTPLIB_NO_EXCEPTIONSinline bool parse_range_header(const std::string &s, Ranges &ranges) {#else@@ -4370,9 +3975,9 @@ inline bool parse_range_header(const std::string &s, Ranges &ranges) try {if (std::regex_match(s, m, re_first_range)) {auto pos = static_cast<size_t>(m.position(1));auto len = static_cast<size_t>(m.length(1));- auto all_valid_ranges = true;+ bool all_valid_ranges = true;split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {- if (!all_valid_ranges) { return; }+ if (!all_valid_ranges) return;static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))");std::cmatch cm;if (std::regex_match(b, e, cm, re_another_range)) {@@ -4417,6 +4022,11 @@ public:bool parse(const char *buf, size_t n, const ContentReceiver &content_callback,const MultipartContentHeader &header_callback) {+ // TODO: support 'filename*'+ static const std::regex re_content_disposition(+ R"~(^Content-Disposition:\s*form-data;\s*name="(.*?)"(?:;\s*filename="(.*?)")?(?:;\s*filename\*=\S+)?\s*$)~",+ std::regex_constants::icase);+buf_append(buf, n);while (buf_size() > 0) {@@ -4449,54 +4059,18 @@ public:break;}+ static const std::string header_name = "content-type:";const auto header = buf_head(pos);-- if (!parse_header(header.data(), header.data() + header.size(),- [&](std::string &&, std::string &&) {})) {- is_valid_ = false;- return false;- }-- static const std::string header_content_type = "Content-Type:";-- if (start_with_case_ignore(header, header_content_type)) {- file_.content_type =- trim_copy(header.substr(header_content_type.size()));+ if (start_with_case_ignore(header, header_name)) {+ file_.content_type = trim_copy(header.substr(header_name.size()));} else {- static const std::regex re_content_disposition(- R"~(^Content-Disposition:\s*form-data;\s*(.*)$)~",- std::regex_constants::icase);-std::smatch m;if (std::regex_match(header, m, re_content_disposition)) {- Params params;- parse_disposition_params(m[1], params);-- auto it = params.find("name");- if (it != params.end()) {- file_.name = it->second;- } else {- is_valid_ = false;- return false;- }-- it = params.find("filename");- if (it != params.end()) { file_.filename = it->second; }-- it = params.find("filename*");- if (it != params.end()) {- // Only allow UTF-8 enconnding...- static const std::regex re_rfc5987_encoding(- R"~(^UTF-8''(.+?)$)~", std::regex_constants::icase);-- std::smatch m2;- if (std::regex_match(it->second, m2, re_rfc5987_encoding)) {- file_.filename = decode_url(m2[1], false); // override...- } else {- is_valid_ = false;- return false;- }- }+ file_.name = m[1];+ file_.filename = m[2];+ } else {+ is_valid_ = false;+ return false;}}buf_erase(pos + crlf_.size());@@ -4534,9 +4108,9 @@ public:buf_erase(crlf_.size());state_ = 1;} else {- if (dash_.size() > buf_size()) { return true; }- if (buf_start_with(dash_)) {- buf_erase(dash_.size());+ if (dash_crlf_.size() > buf_size()) { return true; }+ if (buf_start_with(dash_crlf_)) {+ buf_erase(dash_crlf_.size());is_valid_ = true;buf_erase(buf_size()); // Remove epilogue} else {@@ -4569,6 +4143,7 @@ private:const std::string dash_ = "--";const std::string crlf_ = "\r\n";+ const std::string dash_crlf_ = "--\r\n";std::string boundary_;std::string dash_boundary_crlf_;std::string crlf_dash_boundary_;@@ -4655,30 +4230,26 @@ inline std::string to_lower(const char *beg, const char *end) {return out;}-inline std::string random_string(size_t length) {+inline std::string make_multipart_data_boundary() {static const char data[] ="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";// std::random_device might actually be deterministic on some// platforms, but due to lack of support in the c++ standard library,// doing better requires either some ugly hacks or breaking portability.- static std::random_device seed_gen;+ std::random_device seed_gen;// Request 128 bits of entropy for initialization- static std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(),- seed_gen()};+ std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()};+ std::mt19937 engine(seed_sequence);- static std::mt19937 engine(seed_sequence);+ std::string result = "--cpp-httplib-multipart-data-";- std::string result;- for (size_t i = 0; i < length; i++) {+ for (auto i = 0; i < 16; i++) {result += data[engine() % (sizeof(data) - 1)];}- return result;-}-inline std::string make_multipart_data_boundary() {- return "--cpp-httplib-multipart-data-" + detail::random_string(16);+ return result;}inline bool is_multipart_boundary_chars_valid(const std::string &boundary) {@@ -4733,97 +4304,48 @@ serialize_multipart_formdata(const MultipartFormDataItems &items,body += item.content + serialize_multipart_formdata_item_end();}- if (finish) { body += serialize_multipart_formdata_finish(boundary); }+ if (finish) body += serialize_multipart_formdata_finish(boundary);return body;}-inline bool range_error(Request &req, Response &res) {- if (!req.ranges.empty() && 200 <= res.status && res.status < 300) {- ssize_t contant_len = static_cast<ssize_t>(- res.content_length_ ? res.content_length_ : res.body.size());-- ssize_t prev_first_pos = -1;- ssize_t prev_last_pos = -1;- size_t overwrapping_count = 0;-- // NOTE: The following Range check is based on '14.2. Range' in RFC 9110- // 'HTTP Semantics' to avoid potential denial-of-service attacks.- // https://www.rfc-editor.org/rfc/rfc9110#section-14.2-- // Too many ranges- if (req.ranges.size() > CPPHTTPLIB_RANGE_MAX_COUNT) { return true; }-- for (auto &r : req.ranges) {- auto &first_pos = r.first;- auto &last_pos = r.second;-- if (first_pos == -1 && last_pos == -1) {- first_pos = 0;- last_pos = contant_len;- }-- if (first_pos == -1) {- first_pos = contant_len - last_pos;- last_pos = contant_len - 1;- }-- if (last_pos == -1) { last_pos = contant_len - 1; }-- // Range must be within content length- if (!(0 <= first_pos && first_pos <= last_pos &&- last_pos <= contant_len - 1)) {- return true;- }-- // Ranges must be in ascending order- if (first_pos <= prev_first_pos) { return true; }-- // Request must not have more than two overlapping ranges- if (first_pos <= prev_last_pos) {- overwrapping_count++;- if (overwrapping_count > 2) { return true; }- }+inline std::pair<size_t, size_t>+get_range_offset_and_length(const Request &req, size_t content_length,+ size_t index) {+ auto r = req.ranges[index];- prev_first_pos = (std::max)(prev_first_pos, first_pos);- prev_last_pos = (std::max)(prev_last_pos, last_pos);- }+ if (r.first == -1 && r.second == -1) {+ return std::make_pair(0, content_length);}- return false;-}+ auto slen = static_cast<ssize_t>(content_length);-inline std::pair<size_t, size_t>-get_range_offset_and_length(Range r, size_t content_length) {- (void)(content_length); // patch to get rid of "unused parameter" on release build- assert(r.first != -1 && r.second != -1);- assert(0 <= r.first && r.first < static_cast<ssize_t>(content_length));- assert(r.first <= r.second &&- r.second < static_cast<ssize_t>(content_length));+ if (r.first == -1) {+ r.first = (std::max)(static_cast<ssize_t>(0), slen - r.second);+ r.second = slen - 1;+ }+ if (r.second == -1) { r.second = slen - 1; }return std::make_pair(r.first, static_cast<size_t>(r.second - r.first) + 1);}-inline std::string make_content_range_header_field(- const std::pair<size_t, size_t> &offset_and_length, size_t content_length) {- auto st = offset_and_length.first;- auto ed = st + offset_and_length.second - 1;-+inline std::string make_content_range_header_field(size_t offset, size_t length,+ size_t content_length) {std::string field = "bytes ";- field += std::to_string(st);+ field += std::to_string(offset);field += "-";- field += std::to_string(ed);+ field += std::to_string(offset + length - 1);field += "/";field += std::to_string(content_length);return field;}template <typename SToken, typename CToken, typename Content>-bool process_multipart_ranges_data(const Request &req,+bool process_multipart_ranges_data(const Request &req, Response &res,const std::string &boundary,const std::string &content_type,- size_t content_length, SToken stoken,- CToken ctoken, Content content) {+ SToken stoken, CToken ctoken,+ Content content) {for (size_t i = 0; i < req.ranges.size(); i++) {ctoken("--");stoken(boundary);@@ -4834,51 +4356,50 @@ bool process_multipart_ranges_data(const Request &req,ctoken("\r\n");}- auto offset_and_length =- get_range_offset_and_length(req.ranges[i], content_length);+ auto offsets = get_range_offset_and_length(req, res.body.size(), i);+ auto offset = offsets.first;+ auto length = offsets.second;ctoken("Content-Range: ");- stoken(make_content_range_header_field(offset_and_length, content_length));+ stoken(make_content_range_header_field(offset, length, res.body.size()));ctoken("\r\n");ctoken("\r\n");-- if (!content(offset_and_length.first, offset_and_length.second)) {- return false;- }+ if (!content(offset, length)) { return false; }ctoken("\r\n");}ctoken("--");stoken(boundary);- ctoken("--");+ ctoken("--\r\n");return true;}-inline void make_multipart_ranges_data(const Request &req, Response &res,+inline bool make_multipart_ranges_data(const Request &req, Response &res,const std::string &boundary,const std::string &content_type,- size_t content_length,std::string &data) {- process_multipart_ranges_data(- req, boundary, content_type, content_length,+ return process_multipart_ranges_data(+ req, res, boundary, content_type,[&](const std::string &token) { data += token; },[&](const std::string &token) { data += token; },[&](size_t offset, size_t length) {- assert(offset + length <= content_length);- data += res.body.substr(offset, length);- return true;+ if (offset < res.body.size()) {+ data += res.body.substr(offset, length);+ return true;+ }+ return false;});}-inline size_t get_multipart_ranges_data_length(const Request &req,- const std::string &boundary,- const std::string &content_type,- size_t content_length) {+inline size_t+get_multipart_ranges_data_length(const Request &req, Response &res,+ const std::string &boundary,+ const std::string &content_type) {size_t data_length = 0;process_multipart_ranges_data(- req, boundary, content_type, content_length,+ req, res, boundary, content_type,[&](const std::string &token) { data_length += token.size(); },[&](const std::string &token) { data_length += token.size(); },[&](size_t /*offset*/, size_t length) {@@ -4890,13 +4411,13 @@ inline size_t get_multipart_ranges_data_length(const Request &req,}template <typename T>-inline bool-write_multipart_ranges_data(Stream &strm, const Request &req, Response &res,- const std::string &boundary,- const std::string &content_type,- size_t content_length, const T &is_shutting_down) {+inline bool write_multipart_ranges_data(Stream &strm, const Request &req,+ Response &res,+ const std::string &boundary,+ const std::string &content_type,+ const T &is_shutting_down) {return process_multipart_ranges_data(- req, boundary, content_type, content_length,+ req, res, boundary, content_type,[&](const std::string &token) { strm.write(token); },[&](const std::string &token) { strm.write(token); },[&](size_t offset, size_t length) {@@ -4905,6 +4426,18 @@ write_multipart_ranges_data(Stream &strm, const Request &req, Response &res,});}+inline std::pair<size_t, size_t>+get_range_offset_and_length(const Request &req, const Response &res,+ size_t index) {+ auto r = req.ranges[index];++ if (r.second == -1) {+ r.second = static_cast<ssize_t>(res.content_length_) - 1;+ }++ return std::make_pair(r.first, r.second - r.first + 1);+}+inline bool expect_content(const Request &req) {if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" ||req.method == "PRI" || req.method == "DELETE") {@@ -4938,7 +4471,7 @@ inline std::string message_digest(const std::string &s, const EVP_MD *algo) {std::stringstream ss;for (auto i = 0u; i < hash_length; ++i) {ss << std::hex << std::setw(2) << std::setfill('0')- << static_cast<unsigned int>(hash[i]);+ << (unsigned int)hash[i];}return ss.str();@@ -5031,7 +4564,7 @@ inline bool retrieve_root_certs_from_keychain(CFObjectPtr<CFArrayRef> &certs) {inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) {auto result = false;- for (auto i = 0; i < CFArrayGetCount(certs); ++i) {+ for (int i = 0; i < CFArrayGetCount(certs); ++i) {const auto cert = reinterpret_cast<const __SecCertificate *>(CFArrayGetValueAtIndex(certs, i));@@ -5174,7 +4707,7 @@ inline bool parse_www_authenticate(const Response &res,s = s.substr(pos + 1);auto beg = std::sregex_iterator(s.begin(), s.end(), re);for (auto i = beg; i != std::sregex_iterator(); ++i) {- const auto &m = *i;+ auto m = *i;auto key = s.substr(static_cast<size_t>(m.position(1)),static_cast<size_t>(m.length(1)));auto val = m.length(2) > 0@@ -5191,6 +4724,20 @@ inline bool parse_www_authenticate(const Response &res,return false;}+// https://stackoverflow.com/questions/440133/how-do-i-create-a-random-alpha-numeric-string-in-c/440240#answer-440240+inline std::string random_string(size_t length) {+ auto randchar = []() -> char {+ const char charset[] = "0123456789"+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ "abcdefghijklmnopqrstuvwxyz";+ const size_t max_index = (sizeof(charset) - 1);+ return charset[static_cast<size_t>(std::rand()) % max_index];+ };+ std::string str(length, 0);+ std::generate_n(str.begin(), length, randchar);+ return str;+}+class ContentProviderAdapter {public:explicit ContentProviderAdapter(@@ -5235,7 +4782,7 @@ inline void hosted_at(const std::string &hostname,const auto &addr =*reinterpret_cast<struct sockaddr_storage *>(rp->ai_addr);std::string ip;- auto dummy = -1;+ int dummy = -1;if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip,dummy)) {addrs.push_back(ip);@@ -5255,11 +4802,10 @@ inline std::string append_query_params(const std::string &path,}// Header utilities-inline std::pair<std::string, std::string>-make_range_header(const Ranges &ranges) {+inline std::pair<std::string, std::string> make_range_header(Ranges ranges) {std::string field = "bytes=";auto i = 0;- for (const auto &r : ranges) {+ for (auto r : ranges) {if (i != 0) { field += ", "; }if (r.first != -1) { field += std::to_string(r.first); }field += '-';@@ -5378,7 +4924,7 @@ inline void Response::set_redirect(const std::string &url, int stat) {if (300 <= stat && stat < 400) {this->status = stat;} else {- this->status = StatusCode::Found_302;+ this->status = 302;}}}@@ -5397,22 +4943,13 @@ inline void Response::set_content(const std::string &s,set_content(s.data(), s.size(), content_type);}-inline void Response::set_content(std::string &&s,- const std::string &content_type) {- body = std::move(s);-- auto rng = headers.equal_range("Content-Type");- headers.erase(rng.first, rng.second);- set_header("Content-Type", content_type);-}-inline void Response::set_content_provider(size_t in_length, const std::string &content_type, ContentProvider provider,ContentProviderResourceReleaser resource_releaser) {set_header("Content-Type", content_type);content_length_ = in_length;if (in_length > 0) { content_provider_ = std::move(provider); }- content_provider_resource_releaser_ = std::move(resource_releaser);+ content_provider_resource_releaser_ = resource_releaser;is_chunked_content_provider_ = false;}@@ -5422,7 +4959,7 @@ inline void Response::set_content_provider(set_header("Content-Type", content_type);content_length_ = 0;content_provider_ = detail::ContentProviderAdapter(std::move(provider));- content_provider_resource_releaser_ = std::move(resource_releaser);+ content_provider_resource_releaser_ = resource_releaser;is_chunked_content_provider_ = false;}@@ -5432,7 +4969,7 @@ inline void Response::set_chunked_content_provider(set_header("Content-Type", content_type);content_length_ = 0;content_provider_ = detail::ContentProviderAdapter(std::move(provider));- content_provider_resource_releaser_ = std::move(resource_releaser);+ content_provider_resource_releaser_ = resource_releaser;is_chunked_content_provider_ = true;}@@ -5473,7 +5010,7 @@ inline SocketStream::SocketStream(socket_t sock, time_t read_timeout_sec,write_timeout_sec_(write_timeout_sec),write_timeout_usec_(write_timeout_usec), read_buff_(read_buff_size_, 0) {}-inline SocketStream::~SocketStream() = default;+inline SocketStream::~SocketStream() {}inline bool SocketStream::is_readable() const {return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;@@ -5583,99 +5120,6 @@ inline socket_t BufferStream::socket() const { return 0; }inline const std::string &BufferStream::get_buffer() const { return buffer; }-inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) {- // One past the last ending position of a path param substring- std::size_t last_param_end = 0;--#ifndef CPPHTTPLIB_NO_EXCEPTIONS- // Needed to ensure that parameter names are unique during matcher- // construction- // If exceptions are disabled, only last duplicate path- // parameter will be set- std::unordered_set<std::string> param_name_set;-#endif-- while (true) {- const auto marker_pos = pattern.find(marker, last_param_end);- if (marker_pos == std::string::npos) { break; }-- static_fragments_.push_back(- pattern.substr(last_param_end, marker_pos - last_param_end));-- const auto param_name_start = marker_pos + 1;-- auto sep_pos = pattern.find(separator, param_name_start);- if (sep_pos == std::string::npos) { sep_pos = pattern.length(); }-- auto param_name =- pattern.substr(param_name_start, sep_pos - param_name_start);--#ifndef CPPHTTPLIB_NO_EXCEPTIONS- if (param_name_set.find(param_name) != param_name_set.cend()) {- std::string msg = "Encountered path parameter '" + param_name +- "' multiple times in route pattern '" + pattern + "'.";- throw std::invalid_argument(msg);- }-#endif-- param_names_.push_back(std::move(param_name));-- last_param_end = sep_pos + 1;- }-- if (last_param_end < pattern.length()) {- static_fragments_.push_back(pattern.substr(last_param_end));- }-}--inline bool PathParamsMatcher::match(Request &request) const {- request.matches = std::smatch();- request.path_params.clear();- request.path_params.reserve(param_names_.size());-- // One past the position at which the path matched the pattern last time- std::size_t starting_pos = 0;- for (size_t i = 0; i < static_fragments_.size(); ++i) {- const auto &fragment = static_fragments_[i];-- if (starting_pos + fragment.length() > request.path.length()) {- return false;- }-- // Avoid unnecessary allocation by using strncmp instead of substr +- // comparison- if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(),- fragment.length()) != 0) {- return false;- }-- starting_pos += fragment.length();-- // Should only happen when we have a static fragment after a param- // Example: '/users/:id/subscriptions'- // The 'subscriptions' fragment here does not have a corresponding param- if (i >= param_names_.size()) { continue; }-- auto sep_pos = request.path.find(separator, starting_pos);- if (sep_pos == std::string::npos) { sep_pos = request.path.length(); }-- const auto ¶m_name = param_names_[i];-- request.path_params.emplace(- param_name, request.path.substr(starting_pos, sep_pos - starting_pos));-- // Mark everythin up to '/' as matched- starting_pos = sep_pos + 1;- }- // Returns false if the path is longer than the pattern- return starting_pos >= request.path.length();-}--inline bool RegexMatcher::match(Request &request) const {- request.path_params.clear();- return std::regex_match(request.path, request.matches, regex_);-}-} // namespace detail// HTTP server implementation@@ -5687,72 +5131,69 @@ inline Server::Server()#endif}-inline Server::~Server() = default;--inline std::unique_ptr<detail::MatcherBase>-Server::make_matcher(const std::string &pattern) {- if (pattern.find("/:") != std::string::npos) {- return detail::make_unique<detail::PathParamsMatcher>(pattern);- } else {- return detail::make_unique<detail::RegexMatcher>(pattern);- }-}+inline Server::~Server() {}inline Server &Server::Get(const std::string &pattern, Handler handler) {- get_handlers_.emplace_back(make_matcher(pattern), std::move(handler));+ get_handlers_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Post(const std::string &pattern, Handler handler) {- post_handlers_.emplace_back(make_matcher(pattern), std::move(handler));+ post_handlers_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Post(const std::string &pattern,HandlerWithContentReader handler) {- post_handlers_for_content_reader_.emplace_back(make_matcher(pattern),- std::move(handler));+ post_handlers_for_content_reader_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Put(const std::string &pattern, Handler handler) {- put_handlers_.emplace_back(make_matcher(pattern), std::move(handler));+ put_handlers_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Put(const std::string &pattern,HandlerWithContentReader handler) {- put_handlers_for_content_reader_.emplace_back(make_matcher(pattern),- std::move(handler));+ put_handlers_for_content_reader_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Patch(const std::string &pattern, Handler handler) {- patch_handlers_.emplace_back(make_matcher(pattern), std::move(handler));+ patch_handlers_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Patch(const std::string &pattern,HandlerWithContentReader handler) {- patch_handlers_for_content_reader_.emplace_back(make_matcher(pattern),- std::move(handler));+ patch_handlers_for_content_reader_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Delete(const std::string &pattern, Handler handler) {- delete_handlers_.emplace_back(make_matcher(pattern), std::move(handler));+ delete_handlers_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Delete(const std::string &pattern,HandlerWithContentReader handler) {- delete_handlers_for_content_reader_.emplace_back(make_matcher(pattern),- std::move(handler));+ delete_handlers_for_content_reader_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}inline Server &Server::Options(const std::string &pattern, Handler handler) {- options_handlers_.emplace_back(make_matcher(pattern), std::move(handler));+ options_handlers_.push_back(+ std::make_pair(std::regex(pattern), std::move(handler)));return *this;}@@ -5790,11 +5231,6 @@ Server::set_file_extension_and_mimetype_mapping(const std::string &ext,return *this;}-inline Server &Server::set_default_file_mimetype(const std::string &mime) {- default_file_mimetype_ = mime;- return *this;-}-inline Server &Server::set_file_request_handler(Handler handler) {file_request_handler_ = std::move(handler);return *this;@@ -5836,6 +5272,7 @@ inline Server &Server::set_logger(Logger logger) {inline Server &Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) {expect_100_continue_handler_ = std::move(handler);+return *this;}@@ -5859,12 +5296,6 @@ inline Server &Server::set_default_headers(Headers headers) {return *this;}-inline Server &Server::set_header_writer(- std::function<ssize_t(Stream &, Headers &)> const &writer) {- header_writer_ = writer;- return *this;-}-inline Server &Server::set_keep_alive_max_count(size_t count) {keep_alive_max_count_ = count;return *this;@@ -5900,7 +5331,8 @@ inline Server &Server::set_payload_max_length(size_t length) {inline bool Server::bind_to_port(const std::string &host, int port,int socket_flags) {- return bind_internal(host, port, socket_flags) >= 0;+ if (bind_internal(host, port, socket_flags) < 0) return false;+ return true;}inline int Server::bind_to_any_port(const std::string &host, int socket_flags) {return bind_internal(host, 0, socket_flags);@@ -5934,7 +5366,7 @@ inline void Server::stop() {}}-inline bool Server::parse_request_line(const char *s, Request &req) const {+inline bool Server::parse_request_line(const char *s, Request &req) {auto len = strlen(s);if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') { return false; }len -= 2;@@ -5975,7 +5407,7 @@ inline bool Server::parse_request_line(const char *s, Request &req) const {size_t count = 0;detail::split(req.target.data(), req.target.data() + req.target.size(), '?',- 2, [&](const char *b, const char *e) {+ [&](const char *b, const char *e) {switch (count) {case 0:req.path = detail::decode_url(std::string(b, e), false);@@ -5998,10 +5430,7 @@ inline bool Server::parse_request_line(const char *s, Request &req) const {}inline bool Server::write_response(Stream &strm, bool close_connection,- Request &req, Response &res) {- // NOTE: `req.ranges` should be empty, otherwise it will be applied- // incorrectly to the error content.- req.ranges.clear();+ const Request &req, Response &res) {return write_response_core(strm, close_connection, req, res, false);}@@ -6057,11 +5486,11 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection,detail::BufferStream bstrm;if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status,- status_message(res.status))) {+ detail::status_message(res.status))) {return false;}- if (!header_writer_(bstrm, res.headers)) { return false; }+ if (!detail::write_headers(bstrm, res.headers)) { return false; }// Flush bufferauto &data = bstrm.get_buffer();@@ -6079,6 +5508,7 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection,if (write_content_with_provider(strm, req, res, boundary, content_type)) {res.content_provider_success_ = true;} else {+ res.content_provider_success_ = false;ret = false;}}@@ -6103,16 +5533,15 @@ Server::write_content_with_provider(Stream &strm, const Request &req,return detail::write_content(strm, res.content_provider_, 0,res.content_length_, is_shutting_down);} else if (req.ranges.size() == 1) {- auto offset_and_length = detail::get_range_offset_and_length(- req.ranges[0], res.content_length_);-- return detail::write_content(strm, res.content_provider_,- offset_and_length.first,- offset_and_length.second, is_shutting_down);+ auto offsets =+ detail::get_range_offset_and_length(req, res.content_length_, 0);+ auto offset = offsets.first;+ auto length = offsets.second;+ return detail::write_content(strm, res.content_provider_, offset, length,+ is_shutting_down);} else {return detail::write_multipart_ranges_data(- strm, req, res, boundary, content_type, res.content_length_,- is_shutting_down);+ strm, req, res, boundary, content_type, is_shutting_down);}} else {if (res.is_chunked_content_provider_) {@@ -6169,7 +5598,7 @@ inline bool Server::read_content(Stream &strm, Request &req, Response &res) {const auto &content_type = req.get_header_value("Content-Type");if (!content_type.find("application/x-www-form-urlencoded")) {if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) {- res.status = StatusCode::PayloadTooLarge_413; // NOTE: should be 414?+ res.status = 413; // NOTE: should be 414?return false;}detail::parse_query_text(req.body, req.params);@@ -6188,11 +5617,10 @@ inline bool Server::read_content_with_content_receiver(std::move(multipart_receiver));}-inline bool-Server::read_content_core(Stream &strm, Request &req, Response &res,- ContentReceiver receiver,- MultipartContentHeader multipart_header,- ContentReceiver multipart_receiver) const {+inline bool Server::read_content_core(Stream &strm, Request &req, Response &res,+ ContentReceiver receiver,+ MultipartContentHeader multipart_header,+ ContentReceiver multipart_receiver) {detail::MultipartFormDataParser multipart_form_data_parser;ContentReceiverWithProgress out;@@ -6200,7 +5628,7 @@ Server::read_content_core(Stream &strm, Request &req, Response &res,const auto &content_type = req.get_header_value("Content-Type");std::string boundary;if (!detail::parse_multipart_boundary(content_type, boundary)) {- res.status = StatusCode::BadRequest_400;+ res.status = 400;return false;}@@ -6236,7 +5664,7 @@ Server::read_content_core(Stream &strm, Request &req, Response &res,if (req.is_multipart_form_data()) {if (!multipart_form_data_parser.is_valid()) {- res.status = StatusCode::BadRequest_400;+ res.status = 400;return false;}}@@ -6255,26 +5683,17 @@ inline bool Server::handle_file_request(const Request &req, Response &res,if (path.back() == '/') { path += "index.html"; }if (detail::is_file(path)) {+ detail::read_file(path, res.body);+ auto type =+ detail::find_content_type(path, file_extension_and_mimetype_map_);+ if (type) { res.set_header("Content-Type", type); }for (const auto &kv : entry.headers) {- res.set_header(kv.first, kv.second);+ res.set_header(kv.first.c_str(), kv.second);}-- auto mm = std::make_shared<detail::mmap>(path.c_str());- if (!mm->is_open()) { return false; }-- res.set_content_provider(- mm->size(),- detail::find_content_type(path, file_extension_and_mimetype_map_,- default_file_mimetype_),- [mm](size_t offset, size_t length, DataSink &sink) -> bool {- sink.write(mm->data() + offset, length);- return true;- });-+ res.status = req.has_header("Range") ? 206 : 200;if (!head && file_request_handler_) {file_request_handler_(req, res);}-return true;}}@@ -6370,14 +5789,13 @@ inline bool Server::listen_internal() {#ifdef _WIN32auto timeout = static_cast<uint32_t>(read_timeout_sec_ * 1000 +read_timeout_usec_ / 1000);- setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO,- reinterpret_cast<const char *>(&timeout), sizeof(timeout));+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,+ sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(read_timeout_sec_);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec_);- setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO,- reinterpret_cast<const void *>(&tv), sizeof(tv));+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));#endif}{@@ -6385,22 +5803,17 @@ inline bool Server::listen_internal() {#ifdef _WIN32auto timeout = static_cast<uint32_t>(write_timeout_sec_ * 1000 +write_timeout_usec_ / 1000);- setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO,- reinterpret_cast<const char *>(&timeout), sizeof(timeout));+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,+ sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(write_timeout_sec_);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec_);- setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO,- reinterpret_cast<const void *>(&tv), sizeof(tv));+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));#endif}- if (!task_queue->enqueue(- [this, sock]() { process_and_close_socket(sock); })) {- detail::shutdown_socket(sock);- detail::close_socket(sock);- }+ task_queue->enqueue([this, sock]() { process_and_close_socket(sock); });}task_queue->shutdown();@@ -6416,7 +5829,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) {}// File handler- auto is_head_request = req.method == "HEAD";+ bool is_head_request = req.method == "HEAD";if ((req.method == "GET" || is_head_request) &&handle_file_request(req, res, is_head_request)) {return true;@@ -6482,17 +5895,17 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) {return dispatch_request(req, res, patch_handlers_);}- res.status = StatusCode::BadRequest_400;+ res.status = 400;return false;}inline bool Server::dispatch_request(Request &req, Response &res,- const Handlers &handlers) const {+ const Handlers &handlers) {for (const auto &x : handlers) {- const auto &matcher = x.first;+ const auto &pattern = x.first;const auto &handler = x.second;- if (matcher->match(req)) {+ if (std::regex_match(req.path, req.matches, pattern)) {handler(req, res);return true;}@@ -6502,18 +5915,18 @@ inline bool Server::dispatch_request(Request &req, Response &res,inline void Server::apply_ranges(const Request &req, Response &res,std::string &content_type,- std::string &boundary) const {+ std::string &boundary) {if (req.ranges.size() > 1) {+ boundary = detail::make_multipart_data_boundary();+auto it = res.headers.find("Content-Type");if (it != res.headers.end()) {content_type = it->second;res.headers.erase(it);}- boundary = detail::make_multipart_data_boundary();-- res.set_header("Content-Type",- "multipart/byteranges; boundary=" + boundary);+ res.headers.emplace("Content-Type",+ "multipart/byteranges; boundary=" + boundary);}auto type = detail::encoding_type(req, res);@@ -6524,17 +5937,16 @@ inline void Server::apply_ranges(const Request &req, Response &res,if (req.ranges.empty()) {length = res.content_length_;} else if (req.ranges.size() == 1) {- auto offset_and_length = detail::get_range_offset_and_length(- req.ranges[0], res.content_length_);-- length = offset_and_length.second;-+ auto offsets =+ detail::get_range_offset_and_length(req, res.content_length_, 0);+ auto offset = offsets.first;+ length = offsets.second;auto content_range = detail::make_content_range_header_field(- offset_and_length, res.content_length_);+ offset, length, res.content_length_);res.set_header("Content-Range", content_range);} else {- length = detail::get_multipart_ranges_data_length(- req, boundary, content_type, res.content_length_);+ length = detail::get_multipart_ranges_data_length(req, res, boundary,+ content_type);}res.set_header("Content-Length", std::to_string(length));} else {@@ -6553,22 +5965,28 @@ inline void Server::apply_ranges(const Request &req, Response &res,if (req.ranges.empty()) {;} else if (req.ranges.size() == 1) {- auto offset_and_length =- detail::get_range_offset_and_length(req.ranges[0], res.body.size());- auto offset = offset_and_length.first;- auto length = offset_and_length.second;-+ auto offsets =+ detail::get_range_offset_and_length(req, res.body.size(), 0);+ auto offset = offsets.first;+ auto length = offsets.second;auto content_range = detail::make_content_range_header_field(- offset_and_length, res.body.size());+ offset, length, res.body.size());res.set_header("Content-Range", content_range);-- assert(offset + length <= res.body.size());- res.body = res.body.substr(offset, length);+ if (offset < res.body.size()) {+ res.body = res.body.substr(offset, length);+ } else {+ res.body.clear();+ res.status = 416;+ }} else {std::string data;- detail::make_multipart_ranges_data(req, res, boundary, content_type,- res.body.size(), data);- res.body.swap(data);+ if (detail::make_multipart_ranges_data(req, res, boundary, content_type,+ data)) {+ res.body.swap(data);+ } else {+ res.body.clear();+ res.status = 416;+ }}if (type != detail::EncodingType::None) {@@ -6607,12 +6025,12 @@ inline void Server::apply_ranges(const Request &req, Response &res,inline bool Server::dispatch_request_for_content_reader(Request &req, Response &res, ContentReader content_reader,- const HandlersForContentReader &handlers) const {+ const HandlersForContentReader &handlers) {for (const auto &x : handlers) {- const auto &matcher = x.first;+ const auto &pattern = x.first;const auto &handler = x.second;- if (matcher->match(req)) {+ if (std::regex_match(req.path, req.matches, pattern)) {handler(req, res, content_reader);return true;}@@ -6632,10 +6050,15 @@ Server::process_request(Stream &strm, bool close_connection,if (!line_reader.getline()) { return false; }Request req;-Response res;+res.version = "HTTP/1.1";- res.headers = default_headers_;++ for (const auto &header : default_headers_) {+ if (res.headers.find(header.first) == res.headers.end()) {+ res.headers.insert(header);+ }+ }#ifdef _WIN32// TODO: Increase FD_SETSIZE statically (libzmq), dynamically (MySQL).@@ -6645,7 +6068,7 @@ Server::process_request(Stream &strm, bool close_connection,if (strm.socket() >= FD_SETSIZE) {Headers dummy;detail::read_headers(strm, dummy);- res.status = StatusCode::InternalServerError_500;+ res.status = 500;return write_response(strm, close_connection, req, res);}#endif@@ -6655,14 +6078,14 @@ Server::process_request(Stream &strm, bool close_connection,if (line_reader.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) {Headers dummy;detail::read_headers(strm, dummy);- res.status = StatusCode::UriTooLong_414;+ res.status = 414;return write_response(strm, close_connection, req, res);}// Request line and headersif (!parse_request_line(line_reader.ptr(), req) ||!detail::read_headers(strm, req.headers)) {- res.status = StatusCode::BadRequest_400;+ res.status = 400;return write_response(strm, close_connection, req, res);}@@ -6686,7 +6109,7 @@ Server::process_request(Stream &strm, bool close_connection,if (req.has_header("Range")) {const auto &range_header_value = req.get_header_value("Range");if (!detail::parse_range_header(range_header_value, req.ranges)) {- res.status = StatusCode::RangeNotSatisfiable_416;+ res.status = 416;return write_response(strm, close_connection, req, res);}}@@ -6694,22 +6117,22 @@ Server::process_request(Stream &strm, bool close_connection,if (setup_request) { setup_request(req); }if (req.get_header_value("Expect") == "100-continue") {- int status = StatusCode::Continue_100;+ auto status = 100;if (expect_100_continue_handler_) {status = expect_100_continue_handler_(req, res);}switch (status) {- case StatusCode::Continue_100:- case StatusCode::ExpectationFailed_417:+ case 100:+ case 417:strm.write_format("HTTP/1.1 %d %s\r\n\r\n", status,- status_message(status));+ detail::status_message(status));break;default: return write_response(strm, close_connection, req, res);}}- // Routing- auto routed = false;+ // Rounting+ bool routed = false;#ifdef CPPHTTPLIB_NO_EXCEPTIONSrouted = routing(req, res, strm);#else@@ -6721,7 +6144,7 @@ Server::process_request(Stream &strm, bool close_connection,exception_handler_(req, res, ep);routed = true;} else {- res.status = StatusCode::InternalServerError_500;+ res.status = 500;std::string val;auto s = e.what();for (size_t i = 0; s[i]; i++) {@@ -6739,29 +6162,17 @@ Server::process_request(Stream &strm, bool close_connection,exception_handler_(req, res, ep);routed = true;} else {- res.status = StatusCode::InternalServerError_500;+ res.status = 500;res.set_header("EXCEPTION_WHAT", "UNKNOWN");}}#endif- if (routed) {- if (res.status == -1) {- res.status = req.ranges.empty() ? StatusCode::OK_200- : StatusCode::PartialContent_206;- }-- if (detail::range_error(req, res)) {- res.body.clear();- res.content_length_ = 0;- res.content_provider_ = nullptr;- res.status = StatusCode::RangeNotSatisfiable_416;- return write_response(strm, close_connection, req, res);- }+ if (routed) {+ if (res.status == -1) { res.status = req.ranges.empty() ? 200 : 206; }return write_response_with_content(strm, close_connection, req, res);} else {- if (res.status == -1) { res.status = StatusCode::NotFound_404; }-+ if (res.status == -1) { res.status = 404; }return write_response(strm, close_connection, req, res);}}@@ -6861,7 +6272,7 @@ inline socket_t ClientImpl::create_client_socket(Error &error) const {// Check is custom IP specified for host_std::string ip;auto it = addr_map_.find(host_);- if (it != addr_map_.end()) { ip = it->second; }+ if (it != addr_map_.end()) ip = it->second;return detail::create_client_socket(host_, ip, port_, address_family_, tcp_nodelay_, socket_options_,@@ -6886,7 +6297,7 @@ inline void ClientImpl::shutdown_ssl(Socket & /*socket*/,socket_requests_are_from_thread_ == std::this_thread::get_id());}-inline void ClientImpl::shutdown_socket(Socket &socket) const {+inline void ClientImpl::shutdown_socket(Socket &socket) {if (socket.sock == INVALID_SOCKET) { return; }detail::shutdown_socket(socket.sock);}@@ -6911,7 +6322,7 @@ inline void ClientImpl::close_socket(Socket &socket) {}inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,- Response &res) const {+ Response &res) {std::array<char, 2048> buf{};detail::stream_line_reader line_reader(strm, buf.data(), buf.size());@@ -6919,9 +6330,9 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,if (!line_reader.getline()) { return false; }#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR- const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n");-#elseconst static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n");+#else+ const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n");#endifstd::cmatch m;@@ -6933,7 +6344,7 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,res.reason = std::string(m[3]);// Ignore '100 Continue'- while (res.status == StatusCode::Continue_100) {+ while (res.status == 100) {if (!line_reader.getline()) { return false; } // CRLFif (!line_reader.getline()) { return false; } // next response line@@ -7081,31 +6492,15 @@ inline bool ClientImpl::handle_request(Stream &strm, Request &req,if (!ret) { return false; }- if (res.get_header_value("Connection") == "close" ||- (res.version == "HTTP/1.0" && res.reason != "Connection established")) {- // TODO this requires a not-entirely-obvious chain of calls to be correct- // for this to be safe.-- // This is safe to call because handle_request is only called by send_- // which locks the request mutex during the process. It would be a bug- // to call it from a different thread since it's a thread-safety issue- // to do these things to the socket if another thread is using the socket.- std::lock_guard<std::mutex> guard(socket_mutex_);- shutdown_ssl(socket_, true);- shutdown_socket(socket_);- close_socket(socket_);- }-if (300 < res.status && res.status < 400 && follow_location_) {req = req_save;ret = redirect(req, res, error);}#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- if ((res.status == StatusCode::Unauthorized_401 ||- res.status == StatusCode::ProxyAuthenticationRequired_407) &&+ if ((res.status == 401 || res.status == 407) &&req.authorization_count_ < 5) {- auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407;+ auto is_proxy = res.status == 407;const auto &username =is_proxy ? proxy_digest_auth_username_ : digest_auth_username_;const auto &password =@@ -7176,7 +6571,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {} else {if (next_scheme == "https") {#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- SSLClient cli(next_host, next_port);+ SSLClient cli(next_host.c_str(), next_port);cli.copy_settings(*this);if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); }return detail::redirect(cli, req, res, path, location, error);@@ -7184,7 +6579,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {return false;#endif} else {- ClientImpl cli(next_host, next_port);+ ClientImpl cli(next_host.c_str(), next_port);cli.copy_settings(*this);return detail::redirect(cli, req, res, path, location, error);}@@ -7193,7 +6588,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {inline bool ClientImpl::write_content_with_provider(Stream &strm,const Request &req,- Error &error) const {+ Error &error) {auto is_shutting_down = []() { return false; };if (req.is_chunked_content_provider_) {@@ -7221,32 +6616,32 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,// Prepare additional headersif (close_connection) {if (!req.has_header("Connection")) {- req.set_header("Connection", "close");+ req.headers.emplace("Connection", "close");}}if (!req.has_header("Host")) {if (is_ssl()) {if (port_ == 443) {- req.set_header("Host", host_);+ req.headers.emplace("Host", host_);} else {- req.set_header("Host", host_and_port_);+ req.headers.emplace("Host", host_and_port_);}} else {if (port_ == 80) {- req.set_header("Host", host_);+ req.headers.emplace("Host", host_);} else {- req.set_header("Host", host_and_port_);+ req.headers.emplace("Host", host_and_port_);}}}- if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); }+ if (!req.has_header("Accept")) { req.headers.emplace("Accept", "*/*"); }#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENTif (!req.has_header("User-Agent")) {auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION;- req.set_header("User-Agent", agent);+ req.headers.emplace("User-Agent", agent);}#endif@@ -7255,23 +6650,23 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,if (!req.is_chunked_content_provider_) {if (!req.has_header("Content-Length")) {auto length = std::to_string(req.content_length_);- req.set_header("Content-Length", length);+ req.headers.emplace("Content-Length", length);}}} else {if (req.method == "POST" || req.method == "PUT" ||req.method == "PATCH") {- req.set_header("Content-Length", "0");+ req.headers.emplace("Content-Length", "0");}}} else {if (!req.has_header("Content-Type")) {- req.set_header("Content-Type", "text/plain");+ req.headers.emplace("Content-Type", "text/plain");}if (!req.has_header("Content-Length")) {auto length = std::to_string(req.body.size());- req.set_header("Content-Length", length);+ req.headers.emplace("Content-Length", length);}}@@ -7311,7 +6706,7 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,const auto &path = url_encode_ ? detail::encode_url(req.path) : req.path;bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str());- header_writer_(bstrm, req.headers);+ detail::write_headers(bstrm, req.headers);// Flush bufferauto &data = bstrm.get_buffer();@@ -7339,10 +6734,12 @@ inline std::unique_ptr<Response> ClientImpl::send_with_content_provider(ContentProvider content_provider,ContentProviderWithoutLength content_provider_without_length,const std::string &content_type, Error &error) {- if (!content_type.empty()) { req.set_header("Content-Type", content_type); }+ if (!content_type.empty()) {+ req.headers.emplace("Content-Type", content_type);+ }#ifdef CPPHTTPLIB_ZLIB_SUPPORT- if (compress_) { req.set_header("Content-Encoding", "gzip"); }+ if (compress_) { req.headers.emplace("Content-Encoding", "gzip"); }#endif#ifdef CPPHTTPLIB_ZLIB_SUPPORT@@ -7403,9 +6800,10 @@ inline std::unique_ptr<Response> ClientImpl::send_with_content_provider(req.content_provider_ = detail::ContentProviderAdapter(std::move(content_provider_without_length));req.is_chunked_content_provider_ = true;- req.set_header("Transfer-Encoding", "chunked");+ req.headers.emplace("Transfer-Encoding", "chunked");} else {req.body.assign(body, content_length);+ ;}}@@ -7466,8 +6864,7 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,}// Body- if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" &&- req.method != "CONNECT") {+ if ((res.status != 204) && req.method != "HEAD" && req.method != "CONNECT") {auto redirect = 300 < res.status && res.status < 400 && follow_location_;if (req.response_handler && !redirect) {@@ -7512,6 +6909,24 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,}}+ if (res.get_header_value("Connection") == "close" ||+ (res.version == "HTTP/1.0" && res.reason != "Connection established")) {+ // TODO this requires a not-entirely-obvious chain of calls to be correct+ // for this to be safe. Maybe a code refactor (such as moving this out to+ // the send function and getting rid of the recursiveness of the mutex)+ // could make this more obvious.++ // This is safe to call because process_request is only called by+ // handle_request which is only called by send, which locks the request+ // mutex during the process. It would be a bug to call it from a different+ // thread since it's a thread-safety issue to do these things to the socket+ // if another thread is using the socket.+ std::lock_guard<std::mutex> guard(socket_mutex_);+ shutdown_ssl(socket_, true);+ shutdown_socket(socket_);+ close_socket(socket_);+ }+// Logif (logger_) { logger_(req, res); }@@ -7520,14 +6935,13 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(const std::string &boundary, const MultipartFormDataItems &items,- const MultipartFormDataProviderItems &provider_items) const {- size_t cur_item = 0;- size_t cur_start = 0;+ const MultipartFormDataProviderItems &provider_items) {+ size_t cur_item = 0, cur_start = 0;// cur_item and cur_start are copied to within the std::function and maintain// state between successive callsreturn [&, cur_item, cur_start](size_t offset,DataSink &sink) mutable -> bool {- if (!offset && !items.empty()) {+ if (!offset && items.size()) {sink.os << detail::serialize_multipart_formdata(items, boundary, false);return true;} else if (cur_item < provider_items.size()) {@@ -7540,13 +6954,12 @@ inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(}DataSink cur_sink;- auto has_data = true;+ bool has_data = true;cur_sink.write = sink.write;cur_sink.done = [&]() { has_data = false; };- if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) {+ if (!provider_items[cur_item].provider(offset - cur_start, cur_sink))return false;- }if (!has_data) {sink.os << detail::serialize_multipart_formdata_item_end();@@ -7665,15 +7078,14 @@ inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,if (params.empty()) { return Get(path, headers); }std::string path_with_query = append_query_params(path, params);- return Get(path_with_query, headers, std::move(progress));+ return Get(path_with_query.c_str(), headers, progress);}inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,const Headers &headers,ContentReceiver content_receiver,Progress progress) {- return Get(path, params, headers, nullptr, std::move(content_receiver),- std::move(progress));+ return Get(path, params, headers, nullptr, content_receiver, progress);}inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,@@ -7682,13 +7094,12 @@ inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,ContentReceiver content_receiver,Progress progress) {if (params.empty()) {- return Get(path, headers, std::move(response_handler),- std::move(content_receiver), std::move(progress));+ return Get(path, headers, response_handler, content_receiver, progress);}std::string path_with_query = append_query_params(path, params);- return Get(path_with_query, headers, std::move(response_handler),- std::move(content_receiver), std::move(progress));+ return Get(path_with_query.c_str(), headers, response_handler,+ content_receiver, progress);}inline Result ClientImpl::Head(const std::string &path) {@@ -7790,7 +7201,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,const auto &content_type =detail::serialize_multipart_formdata_get_content_type(boundary);const auto &body = detail::serialize_multipart_formdata(items, boundary);- return Post(path, headers, body, content_type);+ return Post(path, headers, body, content_type.c_str());}inline Result ClientImpl::Post(const std::string &path, const Headers &headers,@@ -7803,7 +7214,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,const auto &content_type =detail::serialize_multipart_formdata_get_content_type(boundary);const auto &body = detail::serialize_multipart_formdata(items, boundary);- return Post(path, headers, body, content_type);+ return Post(path, headers, body, content_type.c_str());}inline Result@@ -8012,7 +7423,9 @@ inline Result ClientImpl::Delete(const std::string &path,req.headers = headers;req.path = path;- if (!content_type.empty()) { req.set_header("Content-Type", content_type); }+ if (!content_type.empty()) {+ req.headers.emplace("Content-Type", content_type);+ }req.body.assign(body, content_length);return send_(std::move(req));@@ -8045,6 +7458,13 @@ inline Result ClientImpl::Options(const std::string &path,return send_(std::move(req));}+inline size_t ClientImpl::is_socket_open() const {+ std::lock_guard<std::mutex> guard(socket_mutex_);+ return socket_.is_open();+}++inline socket_t ClientImpl::socket() const { return socket_.sock; }+inline void ClientImpl::stop() {std::lock_guard<std::mutex> guard(socket_mutex_);@@ -8068,17 +7488,6 @@ inline void ClientImpl::stop() {close_socket(socket_);}-inline std::string ClientImpl::host() const { return host_; }--inline int ClientImpl::port() const { return port_; }--inline size_t ClientImpl::is_socket_open() const {- std::lock_guard<std::mutex> guard(socket_mutex_);- return socket_.is_open();-}--inline socket_t ClientImpl::socket() const { return socket_.sock; }-inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) {connection_timeout_sec_ = sec;connection_timeout_usec_ = usec;@@ -8127,11 +7536,6 @@ inline void ClientImpl::set_default_headers(Headers headers) {default_headers_ = std::move(headers);}-inline void ClientImpl::set_header_writer(- std::function<ssize_t(Stream &, Headers &)> const &writer) {- header_writer_ = writer;-}-inline void ClientImpl::set_address_family(int family) {address_family_ = family;}@@ -8171,7 +7575,9 @@ inline void ClientImpl::set_proxy_digest_auth(const std::string &username,proxy_digest_auth_username_ = username;proxy_digest_auth_password_ = password;}+#endif+#ifdef CPPHTTPLIB_OPENSSL_SUPPORTinline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path,const std::string &ca_cert_dir_path) {ca_cert_file_path_ = ca_cert_file_path;@@ -8183,34 +7589,9 @@ inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) {ca_cert_store_ = ca_cert_store;}}+#endif-inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert,- std::size_t size) const {- auto mem = BIO_new_mem_buf(ca_cert, static_cast<int>(size));- if (!mem) { return nullptr; }-- auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr);- if (!inf) {- BIO_free_all(mem);- return nullptr;- }-- auto cts = X509_STORE_new();- if (cts) {- for (auto i = 0; i < static_cast<int>(sk_X509_INFO_num(inf)); i++) {- auto itmp = sk_X509_INFO_value(inf, i);- if (!itmp) { continue; }-- if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); }- if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); }- }- }-- sk_X509_INFO_pop_free(inf, X509_INFO_free);- BIO_free_all(mem);- return cts;-}-+#ifdef CPPHTTPLIB_OPENSSL_SUPPORTinline void ClientImpl::enable_server_certificate_verification(bool enabled) {server_certificate_verification_ = enabled;}@@ -8274,7 +7655,7 @@ bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl,U ssl_connect_or_accept,time_t timeout_sec,time_t timeout_usec) {- auto res = 0;+ int res = 0;while ((res = ssl_connect_or_accept(ssl)) != 1) {auto err = SSL_get_error(ssl, res);switch (err) {@@ -8337,7 +7718,7 @@ inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl,SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY);}-inline SSLSocketStream::~SSLSocketStream() = default;+inline SSLSocketStream::~SSLSocketStream() {}inline bool SSLSocketStream::is_readable() const {return detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;@@ -8355,7 +7736,7 @@ inline ssize_t SSLSocketStream::read(char *ptr, size_t size) {auto ret = SSL_read(ssl_, ptr, static_cast<int>(size));if (ret < 0) {auto err = SSL_get_error(ssl_, ret);- auto n = 1000;+ int n = 1000;#ifdef _WIN32while (--n >= 0 && (err == SSL_ERROR_WANT_READ ||(err == SSL_ERROR_SYSCALL &&@@ -8388,7 +7769,7 @@ inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) {auto ret = SSL_write(ssl_, ptr, static_cast<int>(handle_size));if (ret < 0) {auto err = SSL_get_error(ssl_, ret);- auto n = 1000;+ int n = 1000;#ifdef _WIN32while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE ||(err == SSL_ERROR_SYSCALL &&@@ -8441,10 +7822,10 @@ inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path,SSL_CTX_set_min_proto_version(ctx_, TLS1_1_VERSION);+ // add default password callback before opening encrypted private keyif (private_key_password != nullptr && (private_key_password[0] != '\0')) {- SSL_CTX_set_default_passwd_cb_userdata(- ctx_,- reinterpret_cast<void *>(const_cast<char *>(private_key_password)));+ SSL_CTX_set_default_passwd_cb_userdata(ctx_,+ (char *)private_key_password);}if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 ||@@ -8546,23 +7927,16 @@ inline SSLClient::SSLClient(const std::string &host, int port)inline SSLClient::SSLClient(const std::string &host, int port,const std::string &client_cert_path,- const std::string &client_key_path,- const std::string &private_key_password)+ const std::string &client_key_path): ClientImpl(host, port, client_cert_path, client_key_path) {ctx_ = SSL_CTX_new(TLS_client_method());detail::split(&host_[0], &host_[host_.size()], '.',[&](const char *b, const char *e) {- host_components_.emplace_back(b, e);+ host_components_.emplace_back(std::string(b, e));});if (!client_cert_path.empty() && !client_key_path.empty()) {- if (!private_key_password.empty()) {- SSL_CTX_set_default_passwd_cb_userdata(- ctx_, reinterpret_cast<void *>(- const_cast<char *>(private_key_password.c_str())));- }-if (SSL_CTX_use_certificate_file(ctx_, client_cert_path.c_str(),SSL_FILETYPE_PEM) != 1 ||SSL_CTX_use_PrivateKey_file(ctx_, client_key_path.c_str(),@@ -8574,23 +7948,16 @@ inline SSLClient::SSLClient(const std::string &host, int port,}inline SSLClient::SSLClient(const std::string &host, int port,- X509 *client_cert, EVP_PKEY *client_key,- const std::string &private_key_password)+ X509 *client_cert, EVP_PKEY *client_key): ClientImpl(host, port) {ctx_ = SSL_CTX_new(TLS_client_method());detail::split(&host_[0], &host_[host_.size()], '.',[&](const char *b, const char *e) {- host_components_.emplace_back(b, e);+ host_components_.emplace_back(std::string(b, e));});if (client_cert != nullptr && client_key != nullptr) {- if (!private_key_password.empty()) {- SSL_CTX_set_default_passwd_cb_userdata(- ctx_, reinterpret_cast<void *>(- const_cast<char *>(private_key_password.c_str())));- }-if (SSL_CTX_use_certificate(ctx_, client_cert) != 1 ||SSL_CTX_use_PrivateKey(ctx_, client_key) != 1) {SSL_CTX_free(ctx_);@@ -8622,11 +7989,6 @@ inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) {}}-inline void SSLClient::load_ca_cert_store(const char *ca_cert,- std::size_t size) {- set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size));-}-inline long SSLClient::get_openssl_verify_result() const {return verify_result_;}@@ -8641,14 +8003,14 @@ inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) {inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,bool &success, Error &error) {success = true;- Response proxy_res;+ Response res2;if (!detail::process_client_socket(socket.sock, read_timeout_sec_, read_timeout_usec_,write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {Request req2;req2.method = "CONNECT";req2.path = host_and_port_;- return process_request(strm, req2, proxy_res, false, error);+ return process_request(strm, req2, res2, false, error);})) {// Thread-safe to close everything because we are assuming there are no// requests in flight@@ -8659,12 +8021,12 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,return false;}- if (proxy_res.status == StatusCode::ProxyAuthenticationRequired_407) {+ if (res2.status == 407) {if (!proxy_digest_auth_username_.empty() &&!proxy_digest_auth_password_.empty()) {std::map<std::string, std::string> auth;- if (detail::parse_www_authenticate(proxy_res, auth, true)) {- proxy_res = Response();+ if (detail::parse_www_authenticate(res2, auth, true)) {+ Response res3;if (!detail::process_client_socket(socket.sock, read_timeout_sec_, read_timeout_usec_,write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {@@ -8675,7 +8037,7 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,req3, auth, 1, detail::random_string(10),proxy_digest_auth_username_, proxy_digest_auth_password_,true));- return process_request(strm, req3, proxy_res, false, error);+ return process_request(strm, req3, res3, false, error);})) {// Thread-safe to close everything because we are assuming there are// no requests in flight@@ -8686,28 +8048,17 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,return false;}}+ } else {+ res = res2;+ return false;}}- // If status code is not 200, proxy request is failed.- // Set error to ProxyConnection and return proxy response- // as the response of the request- if (proxy_res.status != StatusCode::OK_200) {- error = Error::ProxyConnection;- res = std::move(proxy_res);- // Thread-safe to close everything because we are assuming there are- // no requests in flight- shutdown_ssl(socket, true);- shutdown_socket(socket);- close_socket(socket);- return false;- }-return true;}inline bool SSLClient::load_certs() {- auto ret = true;+ bool ret = true;std::call_once(initialize_cert_, [&]() {std::lock_guard<std::mutex> guard(ctx_mutex_);@@ -8783,11 +8134,7 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) {return true;},[&](SSL *ssl2) {- // NOTE: Direct call instead of using the OpenSSL macro to suppress- // -Wold-style-cast warning- // SSL_set_tlsext_host_name(ssl2, host_.c_str());- SSL_ctrl(ssl2, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name,- static_cast<void *>(const_cast<char *>(host_.c_str())));+ SSL_set_tlsext_host_name(ssl2, host_.c_str());return true;});@@ -8861,8 +8208,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {auto type = GEN_DNS;- struct in6_addr addr6 {};- struct in_addr addr {};+ struct in6_addr addr6;+ struct in_addr addr;size_t addr_len = 0;#ifndef __MINGW32__@@ -8887,9 +8234,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {for (decltype(count) i = 0; i < count && !dsn_matched; i++) {auto val = sk_GENERAL_NAME_value(alt_names, i);if (val->type == type) {- auto name =- reinterpret_cast<const char *>(ASN1_STRING_get0_data(val->d.ia5));- auto name_len = static_cast<size_t>(ASN1_STRING_length(val->d.ia5));+ auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5);+ auto name_len = (size_t)ASN1_STRING_length(val->d.ia5);switch (type) {case GEN_DNS: dsn_matched = check_host_name(name, name_len); break;@@ -8907,8 +8253,7 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {if (dsn_matched || ip_matched) { ret = true; }}- GENERAL_NAMES_free(const_cast<STACK_OF(GENERAL_NAME) *>(- reinterpret_cast<const STACK_OF(GENERAL_NAME) *>(alt_names)));+ GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names);return ret;}@@ -8937,7 +8282,7 @@ inline bool SSLClient::check_host_name(const char *pattern,std::vector<std::string> pattern_components;detail::split(&pattern[0], &pattern[pattern_len], '.',[&](const char *b, const char *e) {- pattern_components.emplace_back(b, e);+ pattern_components.emplace_back(std::string(b, e));});if (host_components_.size() != pattern_components.size()) { return false; }@@ -9016,7 +8361,7 @@ inline Client::Client(const std::string &host, int port,: cli_(detail::make_unique<ClientImpl>(host, port, client_cert_path,client_key_path)) {}-inline Client::~Client() = default;+inline Client::~Client() {}inline bool Client::is_valid() const {return cli_ != nullptr && cli_->is_valid();@@ -9076,20 +8421,19 @@ inline Result Client::Get(const std::string &path, const Headers &headers,}inline Result Client::Get(const std::string &path, const Params ¶ms,const Headers &headers, Progress progress) {- return cli_->Get(path, params, headers, std::move(progress));+ return cli_->Get(path, params, headers, progress);}inline Result Client::Get(const std::string &path, const Params ¶ms,const Headers &headers,ContentReceiver content_receiver, Progress progress) {- return cli_->Get(path, params, headers, std::move(content_receiver),- std::move(progress));+ return cli_->Get(path, params, headers, content_receiver, progress);}inline Result Client::Get(const std::string &path, const Params ¶ms,const Headers &headers,ResponseHandler response_handler,ContentReceiver content_receiver, Progress progress) {- return cli_->Get(path, params, headers, std::move(response_handler),- std::move(content_receiver), std::move(progress));+ return cli_->Get(path, params, headers, response_handler, content_receiver,+ progress);}inline Result Client::Head(const std::string &path) { return cli_->Head(path); }@@ -9321,16 +8665,12 @@ inline bool Client::send(Request &req, Response &res, Error &error) {inline Result Client::send(const Request &req) { return cli_->send(req); }-inline void Client::stop() { cli_->stop(); }--inline std::string Client::host() const { return cli_->host(); }--inline int Client::port() const { return cli_->port(); }-inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); }inline socket_t Client::socket() const { return cli_->socket(); }+inline void Client::stop() { cli_->stop(); }+inline voidClient::set_hostname_addr_map(std::map<std::string, std::string> addr_map) {cli_->set_hostname_addr_map(std::move(addr_map));@@ -9340,11 +8680,6 @@ inline void Client::set_default_headers(Headers headers) {cli_->set_default_headers(std::move(headers));}-inline void Client::set_header_writer(- std::function<ssize_t(Stream &, Headers &)> const &writer) {- cli_->set_header_writer(writer);-}-inline void Client::set_address_family(int family) {cli_->set_address_family(family);}@@ -9419,9 +8754,7 @@ inline void Client::enable_server_certificate_verification(bool enabled) {}#endif-inline void Client::set_logger(Logger logger) {- cli_->set_logger(std::move(logger));-}+inline void Client::set_logger(Logger logger) { cli_->set_logger(logger); }#ifdef CPPHTTPLIB_OPENSSL_SUPPORTinline void Client::set_ca_cert_path(const std::string &ca_cert_file_path,@@ -9437,10 +8770,6 @@ inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) {}}-inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) {- set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size));-}-inline long Client::get_openssl_verify_result() const {if (is_ssl_) {return static_cast<SSLClient &>(*cli_).get_openssl_verify_result();diff --git a/llama.cpp/server/oai.h b/llama.cpp/server/oai.hnew file mode 100644index 00000000..cdbff727--- /dev/null+++ b/llama.cpp/server/oai.h@@ -0,0 +1,228 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi++#pragma once++#include <string>+#include <vector>+#include <set>+#include <mutex>+#include <condition_variable>+#include <unordered_map>++#include "llama.cpp/json.h"+#include "utils.h"++#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"++using json = nlohmann::json;++inline static json oaicompat_completion_params_parse(+ const struct llama_model * model,+ const json &body, /* openai api json semantics */+ const std::string &chat_template)+{+ json llama_params;++ llama_params["__oaicompat"] = true;++ // Map OpenAI parameters to llama.cpp parameters+ //+ // For parameters that are defined by the OpenAI documentation (e.g.+ // temperature), we explicitly specify OpenAI's intended default; we+ // need to do that because sometimes OpenAI disagrees with llama.cpp+ //+ // https://platform.openai.com/docs/api-reference/chat/create+ llama_sampling_params default_sparams;+ llama_params["model"] = json_value(body, "model", std::string("unknown"));+ llama_params["prompt"] = format_chat(model, chat_template, body["messages"]);+ llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);+ llama_params["temperature"] = json_value(body, "temperature", 0.0);+ llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);+ llama_params["top_p"] = json_value(body, "top_p", 1.0);+ llama_params["n_predict"] = json_value(body, "max_tokens", -1);+ llama_params["logit_bias"] = json_value(body, "logit_bias",json::object());+ llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);+ llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);+ llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);+ llama_params["stream"] = json_value(body, "stream", false);+ llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);+ llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);+ llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);+ llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);+ llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);+ llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);+ llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);+ llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);++ if (body.count("grammar") != 0) {+ llama_params["grammar"] = json_value(body, "grammar", json::object());+ }++ // Handle 'stop' field+ if (body.contains("stop") && body["stop"].is_string()) {+ llama_params["stop"] = json::array({body["stop"].get<std::string>()});+ } else {+ llama_params["stop"] = json_value(body, "stop", json::array());+ }++ // Ensure there is ChatML-specific end sequence among stop words+ llama_params["stop"].push_back("<|im_end|>");++ return llama_params;+}++inline static json format_final_response_oaicompat(const json &request, const task_result &response, bool streaming = false)+{+ json result = response.result_json;++ bool stopped_word = result.count("stopped_word") != 0;+ bool stopped_eos = json_value(result, "stopped_eos", false);+ int num_tokens_predicted = json_value(result, "tokens_predicted", 0);+ int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);+ std::string content = json_value(result, "content", std::string(""));++ std::string finish_reason = "length";+ if (stopped_word || stopped_eos) {+ finish_reason = "stop";+ }++ json choices =+ streaming ? json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"delta", json::object()}}})+ : json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"message", json{{"content", content},+ {"role", "assistant"}}}}});++ std::time_t t = std::time(0);++ json res =+ json{{"choices", choices},+ {"created", t},+ {"model",+ json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},+ {"object", streaming ? "chat.completion.chunk" : "chat.completion"},+ {"usage",+ json{{"completion_tokens", num_tokens_predicted},+ {"prompt_tokens", num_prompt_tokens},+ {"total_tokens", num_tokens_predicted + num_prompt_tokens}}},+ {"id", gen_chatcmplid()}};++ if (server_verbose) {+ res["__verbose"] = result;+ }++ if (result.contains("completion_probabilities")) {+ res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());+ }++ return res;+}++// return value is vector as there is one case where we might need to generate two responses+inline static std::vector<json> format_partial_response_oaicompat(const task_result &response) {+ json result = response.result_json;++ if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {+ return std::vector<json>({response.result_json});+ }++ bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;+ std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));++ bool stopped_word = json_value(result, "stopped_word", false);+ bool stopped_eos = json_value(result, "stopped_eos", false);+ bool stopped_limit = json_value(result, "stopped_limit", false);+ std::string content = json_value(result, "content", std::string(""));++ std::string finish_reason;+ if (stopped_word || stopped_eos) {+ finish_reason = "stop";+ }+ if (stopped_limit) {+ finish_reason = "length";+ }++ std::time_t t = std::time(0);++ json choices;++ if (!finish_reason.empty()) {+ choices = json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"delta", json::object()}}});+ } else {+ if (first) {+ if (content.empty()) {+ choices = json::array({json{{"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{{"role", "assistant"}}}}});+ } else {+ // We have to send this as two updates to conform to openai behavior+ json initial_ret = json{{"choices", json::array({json{+ {"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{+ {"role", "assistant"}+ }}}})},+ {"created", t},+ {"id", gen_chatcmplid()},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ json second_ret = json{+ {"choices", json::array({json{{"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{+ {"content", content}}}+ }})},+ {"created", t},+ {"id", gen_chatcmplid()},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ return std::vector<json>({initial_ret, second_ret});+ }+ } else {+ // Some idiosyncrasy in task processing logic makes several trailing calls+ // with empty content, we ignore these at the calee site.+ if (content.empty()) {+ return std::vector<json>({json::object()});+ }++ choices = json::array({json{+ {"finish_reason", nullptr},+ {"index", 0},+ {"delta",+ json{+ {"content", content},+ }},+ }});+ }+ }++ json ret = json{{"choices", choices},+ {"created", t},+ {"id", gen_chatcmplid()},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ return std::vector<json>({ret});+}++inline static json format_embeddings_response_oaicompat(const json &request, const json &embeddings)+{+ json res =+ json{+ {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},+ {"object", "list"},+ {"usage",+ json{{"prompt_tokens", 0},+ {"total_tokens", 0}}},+ {"data", embeddings}+ };+ return res;+}+diff --git a/llama.cpp/server/public/completion.js b/llama.cpp/server/public/completion.jsindex 835ce6e6..ab38a7b4 100644--- a/llama.cpp/server/public/completion.js+++ b/llama.cpp/server/public/completion.js@@ -96,18 +96,18 @@ export async function* llama(prompt, params = {}, config = {}) {}}if (result.error) {- try {- result.error = JSON.parse(result.error);- if (result.error.message.includes('slot unavailable')) {- // Throw an error to be caught by upstream callers- throw new Error('slot unavailable');- } else {- console.error(`llama.cpp error [${result.error.code} - ${result.error.type}]: ${result.error.message}`);- }- } catch(e) {- console.error(`llama.cpp error ${result.error}`)+ result.error = JSON.parse(result.error);+ if (result.error.content.includes('slot unavailable')) {+ // Throw an error to be caught by upstream callers+ throw new Error('slot unavailable');+ } else {+ console.error(`llama.cpp error: ${result.error.content}`);}}+ if (result.error) {+ result.error = JSON.parse(result.error);+ console.error(`llama.cpp error: ${result.error.content}`);+ }}}}diff --git a/llama.cpp/server/public/index.html b/llama.cpp/server/public/index.htmlindex 18f95f19..47cb5f76 100644--- a/llama.cpp/server/public/index.html+++ b/llama.cpp/server/public/index.html@@ -630,16 +630,14 @@const grammarJsonSchemaPropOrder = signal('')const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value- const convertJSONSchemaGrammar = async () => {+ const convertJSONSchemaGrammar = () => {try {- let schema = JSON.parse(params.value.grammar)- const converter = new SchemaConverter({- prop_order: grammarJsonSchemaPropOrder.value+ const schema = JSON.parse(params.value.grammar)+ const converter = new SchemaConverter(+ grammarJsonSchemaPropOrder.value.split(',')- .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),- allow_fetch: true,- })- schema = await converter.resolveRefs(schema, 'input')+ .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {})+ )converter.visit(schema, '')params.value = {...params.value,diff --git a/llama.cpp/server/public/index.js b/llama.cpp/server/public/index.jsindex aa3ffb5e..9db5a9d9 100644--- a/llama.cpp/server/public/index.js+++ b/llama.cpp/server/public/index.js@@ -1 +1 @@-function t(){throw new Error("Cycle detected")}const n=Symbol.for("preact-signals");function e(){if(f>1){f--;return}let t,n=!1;while(void 0!==o){let _=o;o=void 0;l++;while(void 0!==_){const i=_.o;_.o=void 0;_.f&=-3;if(!(8&_.f)&&p(_))try{_.c()}catch(e){if(!n){t=e;n=!0}}_=i}}l=0;f--;if(n)throw t}function _(t){if(f>0)return t();f++;try{return t()}finally{e()}}let i,o,r=0;function u(t){if(r>0)return t();const n=i;i=void 0;r++;try{return t()}finally{r--;i=n}}let f=0,l=0,s=0;function c(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function h(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}h.prototype.brand=n;h.prototype.h=function(){return!0};h.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};h.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};h.prototype.subscribe=function(t){const n=this;return w((function(){const e=n.value,_=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=_}}))};h.prototype.valueOf=function(){return this.value};h.prototype.toString=function(){return this.value+""};h.prototype.toJSON=function(){return this.value};h.prototype.peek=function(){return this.v};Object.defineProperty(h.prototype,"value",{get(){const t=c(this);if(void 0!==t)t.i=this.i;return this.v},set(n){if(i instanceof y)!function(){throw new Error("Computed cannot have side-effects")}();if(n!==this.v){if(l>100)t();this.v=n;this.i++;s++;f++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{e()}}}});function a(t){return new h(t)}function p(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function d(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function v(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function y(t){h.call(this,void 0);this.x=t;this.s=void 0;this.g=s-1;this.f=4}(y.prototype=new h).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===s)return!0;this.g=s;this.f|=1;if(this.i>0&&!p(this)){this.f&=-2;return!0}const t=i;try{d(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;v(this);this.f&=-2;return!0};y.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}h.prototype.S.call(this,t)};y.prototype.U=function(t){if(void 0!==this.t){h.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};y.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};y.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(y.prototype,"value",{get(){if(1&this.f)t();const n=c(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function m(t){return new y(t)}function g(t){const n=t.u;t.u=void 0;if("function"==typeof n){f++;const _=i;i=void 0;try{n()}catch(n){t.f&=-2;t.f|=8;b(t);throw n}finally{i=_;e()}}}function b(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;g(t)}function k(t){if(i!==this)throw new Error("Out-of-order effect");v(this);i=t;this.f&=-2;if(8&this.f)b(this);e()}function S(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}S.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};S.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;g(this);d(this);f++;const n=i;i=this;return k.bind(this,n)};S.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=o;o=this}};S.prototype.d=function(){this.f|=8;if(!(1&this.f))b(this)};function w(t){const n=new S(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,C,E,U,H,P,N,$,D,T={},V=[],A=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,F=Array.isArray;function M(t,n){for(var e in n)t[e]=n[e];return t}function W(t){var n=t.parentNode;n&&n.removeChild(t)}function L(t,n,e){var _,i,o,r={};for(o in n)"key"==o?_=n[o]:"ref"==o?i=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,_,i,null)}function O(t,n,e,_,i){var o={type:t,props:n,key:e,ref:_,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,constructor:void 0,__v:null==i?++E:i,__i:-1,__u:0};return null==i&&null!=C.vnode&&C.vnode(o),o}function R(){return{current:null}}function j(t){return t.children}function I(t,n){this.props=t,this.context=n}function q(t,n){if(null==n)return t.__?q(t.__,t.__i+1):null;for(var e;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e)return e.__e;return"function"==typeof t.type?q(t):null}function B(t,n,e){var _,i=t.__v,o=i.__e,r=t.__P;if(r)return(_=M({},i)).__v=i.__v+1,C.vnode&&C.vnode(_),it(r,_,i,t.__n,void 0!==r.ownerSVGElement,32&i.__u?[o]:null,n,null==o?q(i):o,!!(32&i.__u),e),_.__v=i.__v,_.__.__k[_.__i]=_,_.__d=void 0,_.__e!=o&&G(_),_}function G(t){var n,e;if(null!=(t=t.__)&&null!=t.__c){for(t.__e=t.__c.base=null,n=0;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e){t.__e=t.__c.base=e.__e;break}return G(t)}}function z(t){(!t.__d&&(t.__d=!0)&&H.push(t)&&!J.__r++||P!==C.debounceRendering)&&((P=C.debounceRendering)||N)(J)}function J(){var t,n,e,_=[],i=[];for(H.sort($);t=H.shift();)t.__d&&(e=H.length,n=B(t,_,i)||n,0===e||H.length>e?(ot(_,n,i),i.length=_.length=0,n=void 0,H.sort($)):n&&C.__c&&C.__c(n,V));n&&ot(_,n,i),J.__r=0}function K(t,n,e,_,i,o,r,u,f,l,s){var c,h,a,p,d,v=_&&_.__k||V,y=n.length;for(e.__d=f,Q(e,n,v),f=e.__d,c=0;c<y;c++)null!=(a=e.__k[c])&&"boolean"!=typeof a&&"function"!=typeof a&&(h=-1===a.__i?T:v[a.__i]||T,a.__i=c,it(t,a,h,i,o,r,u,f,l,s),p=a.__e,a.ref&&h.ref!=a.ref&&(h.ref&&ut(h.ref,null,a),s.push(a.ref,a.__c||p,a)),null==d&&null!=p&&(d=p),65536&a.__u||h.__k===a.__k?f=X(a,f,t):"function"==typeof a.type&&void 0!==a.__d?f=a.__d:p&&(f=p.nextSibling),a.__d=void 0,a.__u&=-196609);e.__d=f,e.__e=d}function Q(t,n,e){var _,i,o,r,u,f=n.length,l=e.length,s=l,c=0;for(t.__k=[],_=0;_<f;_++)r=_+c,null!=(i=t.__k[_]=null==(i=n[_])||"boolean"==typeof i||"function"==typeof i?null:"string"==typeof i||"number"==typeof i||"bigint"==typeof i||i.constructor==String?O(null,i,null,null,null):F(i)?O(j,{children:i},null,null,null):void 0===i.constructor&&i.__b>0?O(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):i)?(i.__=t,i.__b=t.__b+1,u=Z(i,e,r,s),i.__i=u,o=null,-1!==u&&(s--,(o=e[u])&&(o.__u|=131072)),null==o||null===o.__v?(-1==u&&c--,"function"!=typeof i.type&&(i.__u|=65536)):u!==r&&(u===r+1?c++:u>r?s>f-r?c+=u-r:c--:u<r?u==r-1&&(c=u-r):c=0,u!==_+c&&(i.__u|=65536))):(o=e[r])&&null==o.key&&o.__e&&0==(131072&o.__u)&&(o.__e==t.__d&&(t.__d=q(o)),ft(o,o,!1),e[r]=null,s--);if(s)for(_=0;_<l;_++)null!=(o=e[_])&&0==(131072&o.__u)&&(o.__e==t.__d&&(t.__d=q(o)),ft(o,o))}function X(t,n,e){var _,i;if("function"==typeof t.type){for(_=t.__k,i=0;_&&i<_.length;i++)_[i]&&(_[i].__=t,n=X(_[i],n,e));return n}t.__e!=n&&(e.insertBefore(t.__e,n||null),n=t.__e);do{n=n&&n.nextSibling}while(null!=n&&8===n.nodeType);return n}function Y(t,n){return n=n||[],null==t||"boolean"==typeof t||(F(t)?t.some((function(t){Y(t,n)})):n.push(t)),n}function Z(t,n,e,_){var i=t.key,o=t.type,r=e-1,u=e+1,f=n[e];if(null===f||f&&i==f.key&&o===f.type&&0==(131072&f.__u))return e;if(_>(null!=f&&0==(131072&f.__u)?1:0))for(;r>=0||u<n.length;){if(r>=0){if((f=n[r])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return r;r--}if(u<n.length){if((f=n[u])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return u;u++}}return-1}function tt(t,n,e){"-"===n[0]?t.setProperty(n,null==e?"":e):t[n]=null==e?"":"number"!=typeof e||A.test(n)?e:e+"px"}function nt(t,n,e,_,i){var o;t:if("style"===n)if("string"==typeof e)t.style.cssText=e;else{if("string"==typeof _&&(t.style.cssText=_=""),_)for(n in _)e&&n in e||tt(t.style,n,"");if(e)for(n in e)_&&e[n]===_[n]||tt(t.style,n,e[n])}else if("o"===n[0]&&"n"===n[1])o=n!==(n=n.replace(/(PointerCapture)$|Capture$/i,"$1")),n=n.toLowerCase()in t?n.toLowerCase().slice(2):n.slice(2),t.l||(t.l={}),t.l[n+o]=e,e?_?e.u=_.u:(e.u=Date.now(),t.addEventListener(n,o?_t:et,o)):t.removeEventListener(n,o?_t:et,o);else{if(i)n=n.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if("width"!==n&&"height"!==n&&"href"!==n&&"list"!==n&&"form"!==n&&"tabIndex"!==n&&"download"!==n&&"rowSpan"!==n&&"colSpan"!==n&&"role"!==n&&n in t)try{t[n]=null==e?"":e;break t}catch(t){}"function"==typeof e||(null==e||!1===e&&"-"!==n[4]?t.removeAttribute(n):t.setAttribute(n,e))}}function et(t){if(this.l){var n=this.l[t.type+!1];if(t.t){if(t.t<=n.u)return}else t.t=Date.now();return n(C.event?C.event(t):t)}}function _t(t){if(this.l)return this.l[t.type+!0](C.event?C.event(t):t)}function it(t,n,e,_,i,o,r,u,f,l){var s,c,h,a,p,d,v,y,m,g,b,k,S,w,x,E=n.type;if(void 0!==n.constructor)return null;128&e.__u&&(f=!!(32&e.__u),o=[u=n.__e=e.__e]),(s=C.__b)&&s(n);t:if("function"==typeof E)try{if(y=n.props,m=(s=E.contextType)&&_[s.__c],g=s?m?m.props.value:s.__:_,e.__c?v=(c=n.__c=e.__c).__=c.__E:("prototype"in E&&E.prototype.render?n.__c=c=new E(y,g):(n.__c=c=new I(y,g),c.constructor=E,c.render=lt),m&&m.sub(c),c.props=y,c.state||(c.state={}),c.context=g,c.__n=_,h=c.__d=!0,c.__h=[],c._sb=[]),null==c.__s&&(c.__s=c.state),null!=E.getDerivedStateFromProps&&(c.__s==c.state&&(c.__s=M({},c.__s)),M(c.__s,E.getDerivedStateFromProps(y,c.__s))),a=c.props,p=c.state,c.__v=n,h)null==E.getDerivedStateFromProps&&null!=c.componentWillMount&&c.componentWillMount(),null!=c.componentDidMount&&c.__h.push(c.componentDidMount);else{if(null==E.getDerivedStateFromProps&&y!==a&&null!=c.componentWillReceiveProps&&c.componentWillReceiveProps(y,g),!c.__e&&(null!=c.shouldComponentUpdate&&!1===c.shouldComponentUpdate(y,c.__s,g)||n.__v===e.__v)){for(n.__v!==e.__v&&(c.props=y,c.state=c.__s,c.__d=!1),n.__e=e.__e,n.__k=e.__k,n.__k.forEach((function(t){t&&(t.__=n)})),b=0;b<c._sb.length;b++)c.__h.push(c._sb[b]);c._sb=[],c.__h.length&&r.push(c);break t}null!=c.componentWillUpdate&&c.componentWillUpdate(y,c.__s,g),null!=c.componentDidUpdate&&c.__h.push((function(){c.componentDidUpdate(a,p,d)}))}if(c.context=g,c.props=y,c.__P=t,c.__e=!1,k=C.__r,S=0,"prototype"in E&&E.prototype.render){for(c.state=c.__s,c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),w=0;w<c._sb.length;w++)c.__h.push(c._sb[w]);c._sb=[]}else do{c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),c.state=c.__s}while(c.__d&&++S<25);c.state=c.__s,null!=c.getChildContext&&(_=M(M({},_),c.getChildContext())),h||null==c.getSnapshotBeforeUpdate||(d=c.getSnapshotBeforeUpdate(a,p)),K(t,F(x=null!=s&&s.type===j&&null==s.key?s.props.children:s)?x:[x],n,e,_,i,o,r,u,f,l),c.base=n.__e,n.__u&=-161,c.__h.length&&r.push(c),v&&(c.__E=c.__=null)}catch(t){n.__v=null,f||null!=o?(n.__e=u,n.__u|=f?160:32,o[o.indexOf(u)]=null):(n.__e=e.__e,n.__k=e.__k),C.__e(t,n,e)}else null==o&&n.__v===e.__v?(n.__k=e.__k,n.__e=e.__e):n.__e=rt(e.__e,n,e,_,i,o,r,f,l);(s=C.diffed)&&s(n)}function ot(t,n,e){for(var _=0;_<e.length;_++)ut(e[_],e[++_],e[++_]);C.__c&&C.__c(n,t),t.some((function(n){try{t=n.__h,n.__h=[],t.some((function(t){t.call(n)}))}catch(t){C.__e(t,n.__v)}}))}function rt(t,n,e,_,i,o,r,u,f){var l,s,c,h,a,p,d,v=e.props,y=n.props,m=n.type;if("svg"===m&&(i=!0),null!=o)for(l=0;l<o.length;l++)if((a=o[l])&&"setAttribute"in a==!!m&&(m?a.localName===m:3===a.nodeType)){t=a,o[l]=null;break}if(null==t){if(null===m)return document.createTextNode(y);t=i?document.createElementNS("http://www.w3.org/2000/svg",m):document.createElement(m,y.is&&y),o=null,u=!1}if(null===m)v===y||u&&t.data===y||(t.data=y);else{if(o=o&&x.call(t.childNodes),v=e.props||T,!u&&null!=o)for(v={},l=0;l<t.attributes.length;l++)v[(a=t.attributes[l]).name]=a.value;for(l in v)a=v[l],"children"==l||("dangerouslySetInnerHTML"==l?c=a:"key"===l||l in y||nt(t,l,null,a,i));for(l in y)a=y[l],"children"==l?h=a:"dangerouslySetInnerHTML"==l?s=a:"value"==l?p=a:"checked"==l?d=a:"key"===l||u&&"function"!=typeof a||v[l]===a||nt(t,l,a,v[l],i);if(s)u||c&&(s.__html===c.__html||s.__html===t.innerHTML)||(t.innerHTML=s.__html),n.__k=[];else if(c&&(t.innerHTML=""),K(t,F(h)?h:[h],n,e,_,i&&"foreignObject"!==m,o,r,o?o[0]:e.__k&&q(e,0),u,f),null!=o)for(l=o.length;l--;)null!=o[l]&&W(o[l]);u||(l="value",void 0!==p&&(p!==t[l]||"progress"===m&&!p||"option"===m&&p!==v[l])&&nt(t,l,p,v[l],!1),l="checked",void 0!==d&&d!==t[l]&&nt(t,l,d,v[l],!1))}return t}function ut(t,n,e){try{"function"==typeof t?t(n):t.current=n}catch(t){C.__e(t,e)}}function ft(t,n,e){var _,i;if(C.unmount&&C.unmount(t),(_=t.ref)&&(_.current&&_.current!==t.__e||ut(_,null,n)),null!=(_=t.__c)){if(_.componentWillUnmount)try{_.componentWillUnmount()}catch(t){C.__e(t,n)}_.base=_.__P=null,t.__c=void 0}if(_=t.__k)for(i=0;i<_.length;i++)_[i]&&ft(_[i],n,e||"function"!=typeof t.type);e||null==t.__e||W(t.__e),t.__=t.__e=t.__d=void 0}function lt(t,n,e){return this.constructor(t,e)}function st(t,n,e){var _,i,o,r;C.__&&C.__(t,n),i=(_="function"==typeof e)?null:e&&e.__k||n.__k,o=[],r=[],it(n,t=(!_&&e||n).__k=L(j,null,[t]),i||T,T,void 0!==n.ownerSVGElement,!_&&e?[e]:i?null:n.firstChild?x.call(n.childNodes):null,o,!_&&e?e:i?i.__e:n.firstChild,_,r),t.__d=void 0,ot(o,t,r)}function ct(t,n){st(t,n,ct)}function ht(t,n,e){var _,i,o,r,u=M({},t.props);for(o in t.type&&t.type.defaultProps&&(r=t.type.defaultProps),n)"key"==o?_=n[o]:"ref"==o?i=n[o]:u[o]=void 0===n[o]&&void 0!==r?r[o]:n[o];return arguments.length>2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,_||t.key,i||t.ref,null)}function at(t,n){var e={__c:n="__cC"+D++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,_;return this.getChildContext||(e=[],(_={})[n]=this,this.getChildContext=function(){return _},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,z(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=V.slice,C={__e:function(t,n,e,_){for(var i,o,r;n=n.__;)if((i=n.__c)&&!i.__)try{if((o=i.constructor)&&null!=o.getDerivedStateFromError&&(i.setState(o.getDerivedStateFromError(t)),r=i.__d),null!=i.componentDidCatch&&(i.componentDidCatch(t,_||{}),r=i.__d),r)return i.__E=i}catch(n){t=n}throw t}},E=0,U=function(t){return null!=t&&null==t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=M({},this.state),"function"==typeof t&&(t=t(M({},e),this.props)),t&&M(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),z(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),z(this))},I.prototype.render=j,H=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,$=function(t,n){return t.__v.__b-n.__v.__b},J.__r=0,D=0;var pt,dt,vt,yt,mt=0,gt=[],bt=[],kt=C,St=kt.__b,wt=kt.__r,xt=kt.diffed,Ct=kt.__c,Et=kt.unmount,Ut=kt.__;function Ht(t,n){kt.__h&&kt.__h(dt,t,mt||n),mt=0;var e=dt.__H||(dt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:bt}),e.__[t]}function Pt(t){return mt=1,Nt(zt,t)}function Nt(t,n,e){var _=Ht(pt++,2);if(_.t=t,!_.__c&&(_.__=[e?e(n):zt(void 0,n),function(t){var n=_.__N?_.__N[0]:_.__[0],e=_.t(n,t);n!==e&&(_.__N=[e,_.__[1]],_.__c.setState({}))}],_.__c=dt,!dt.u)){var i=function(t,n,e){if(!_.__c.__H)return!0;var i=_.__c.__H.__.filter((function(t){return!!t.__c}));if(i.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return i.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&_.__c.props===t)&&(!o||o.call(this,t,n,e))};dt.u=!0;var o=dt.shouldComponentUpdate,r=dt.componentWillUpdate;dt.componentWillUpdate=function(t,n,e){if(this.__e){var _=o;o=void 0,i(t,n,e),o=_}r&&r.call(this,t,n,e)},dt.shouldComponentUpdate=i}return _.__N||_.__}function $t(t,n){var e=Ht(pt++,3);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__H.__h.push(e))}function Dt(t,n){var e=Ht(pt++,4);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__h.push(e))}function Tt(t){return mt=5,At((function(){return{current:t}}),[])}function Vt(t,n,e){mt=6,Dt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function At(t,n){var e=Ht(pt++,7);return Gt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Ft(t,n){return mt=8,At((function(){return t}),n)}function Mt(t){var n=dt.context[t.__c],e=Ht(pt++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(dt)),n.props.value):t.__}function Wt(t,n){kt.useDebugValue&&kt.useDebugValue(n?n(t):t)}function Lt(t){var n=Ht(pt++,10),e=Pt();return n.__=t,dt.componentDidCatch||(dt.componentDidCatch=function(t,_){n.__&&n.__(t,_),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Ot(){var t=Ht(pt++,11);if(!t.__){for(var n=dt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Rt(){for(var t;t=gt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(qt),t.__H.__h.forEach(Bt),t.__H.__h=[]}catch(n){t.__H.__h=[],kt.__e(n,t.__v)}}kt.__b=function(t){dt=null,St&&St(t)},kt.__=function(t,n){t&&n.__k&&n.__k.__m&&(t.__m=n.__k.__m),Ut&&Ut(t,n)},kt.__r=function(t){wt&&wt(t),pt=0;var n=(dt=t.__c).__H;n&&(vt===dt?(n.__h=[],dt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=bt,t.__N=t.i=void 0}))):(n.__h.forEach(qt),n.__h.forEach(Bt),n.__h=[],pt=0)),vt=dt},kt.diffed=function(t){xt&&xt(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==gt.push(n)&&yt===kt.requestAnimationFrame||((yt=kt.requestAnimationFrame)||It)(Rt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==bt&&(t.__=t.__V),t.i=void 0,t.__V=bt}))),vt=dt=null},kt.__c=function(t,n){n.some((function(t){try{t.__h.forEach(qt),t.__h=t.__h.filter((function(t){return!t.__||Bt(t)}))}catch(u){n.some((function(t){t.__h&&(t.__h=[])})),n=[],kt.__e(u,t.__v)}})),Ct&&Ct(t,n)},kt.unmount=function(t){Et&&Et(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{qt(t)}catch(t){n=t}})),e.__H=void 0,n&&kt.__e(n,e.__v))};var jt="function"==typeof requestAnimationFrame;function It(t){var n,e=function(){clearTimeout(_),jt&&cancelAnimationFrame(n),setTimeout(t)},_=setTimeout(e,100);jt&&(n=requestAnimationFrame(e))}function qt(t){var n=dt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),dt=n}function Bt(t){var n=dt;t.__c=t.__(),dt=n}function Gt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function zt(t,n){return"function"==typeof n?n(t):n}function Jt(t,n){C[t]=n.bind(null,C[t]||(()=>{}))}let Kt,Qt;function Xt(t){if(Qt)Qt();Qt=t&&t.S()}function Yt({data:t}){const n=tn(t);n.value=t;const e=At(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!U(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return m(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Yt.displayName="_st";Object.defineProperties(h.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Yt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});Jt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let _ in e){if("children"===_)continue;let i=e[_];if(i instanceof h){if(!t)n.__np=t={};t[_]=i;e[_]=i.peek()}}}t(n)});Jt("__r",(t,n)=>{Xt();let e,_=n.__c;if(_){_.__$f&=-2;e=_.__$u;if(void 0===e)_.__$u=e=function(t){let n;w((function(){n=this}));n.c=()=>{_.__$f|=1;_.setState({})};return n}()}Kt=_;Xt(e);t(n)});Jt("__e",(t,n,e,_)=>{Xt();Kt=void 0;t(n,e,_)});Jt("diffed",(t,n)=>{Xt();Kt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,_=n.props;if(t){let n=e.U;if(n)for(let e in n){let _=n[e];if(void 0!==_&&!(e in t)){_.d();n[e]=void 0}}else{n={};e.U=n}for(let i in t){let o=n[i],r=t[i];if(void 0===o){o=Zt(e,i,r,_);n[i]=o}else o.o(r,_)}}}t(n)});function Zt(t,n,e,_){const i=n in t&&void 0===t.ownerSVGElement,o=a(e);return{o:(t,n)=>{o.value=t;_=n},d:w(()=>{const e=o.value.value;if(_[n]!==e){_[n]=e;if(i)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}Jt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});Jt("__h",(t,n,e,_)=>{if(_<3||9===_)n.__$f|=2;t(n,e,_)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let _ in n)return!0;for(let _ in t)if("__source"!==_&&t[_]!==this.props[_])return!0;for(let _ in this.props)if(!(_ in t))return!0;return!1};function tn(t){return At(()=>a(t),[])}function nn(t){const n=Tt(t);n.current=t;Kt.__$f|=4;return At(()=>m(()=>n.current()),[])}function en(t){const n=Tt(t);n.current=t;$t(()=>w(()=>n.current()),[])}var _n=function(t,n,e,_){var i;n[0]=0;for(var o=1;o<n.length;o++){var r=n[o++],u=n[o]?(n[0]|=r?1:2,e[n[o++]]):n[++o];3===r?_[0]=u:4===r?_[1]=Object.assign(_[1]||{},u):5===r?(_[1]=_[1]||{})[n[++o]]=u:6===r?_[1][n[++o]]+=u+"":r?(i=t.apply(u,_n(t,u,e,["",null])),_.push(i),u[0]?n[0]|=2:(n[o-2]=0,n[o]=i)):_.push(u)}return _},on=new Map;function rn(t){var n=on.get(this);return n||(n=new Map,on.set(this,n)),(n=_n(this,n.get(t)||(n.set(t,n=function(t){for(var n,e,_=1,i="",o="",r=[0],u=function(t){1===_&&(t||(i=i.replace(/^\s*\n\s*|\s*\n\s*$/g,"")))?r.push(0,t,i):3===_&&(t||i)?(r.push(3,t,i),_=2):2===_&&"..."===i&&t?r.push(4,t,0):2===_&&i&&!t?r.push(5,0,!0,i):_>=5&&((i||!t&&5===_)&&(r.push(_,0,i,e),_=6),t&&(r.push(_,t,0,e),_=6)),i=""},f=0;f<t.length;f++){f&&(1===_&&u(),u(f));for(var l=0;l<t[f].length;l++)n=t[f][l],1===_?"<"===n?(u(),r=[r],_=3):i+=n:4===_?"--"===i&&">"===n?(_=1,i=""):i=n+i[0]:o?n===o?o="":i+=n:'"'===n||"'"===n?o=n:">"===n?(u(),_=1):_&&("="===n?(_=5,e=i,i=""):"/"===n&&(_<5||">"===t[f][l+1])?(u(),3===_&&(r=r[0]),_=r,(r=r[0]).push(2,0,_),_=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),_=2):i+=n),3===_&&"!--"===i&&(_=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var un=rn.bind(L);export{I as Component,j as Fragment,h as Signal,_ as batch,ht as cloneElement,m as computed,at as createContext,L as createElement,R as createRef,w as effect,L as h,un as html,ct as hydrate,U as isValidElement,C as options,st as render,a as signal,Y as toChildArray,u as untracked,Ft as useCallback,nn as useComputed,Mt as useContext,Wt as useDebugValue,$t as useEffect,Lt as useErrorBoundary,Ot as useId,Vt as useImperativeHandle,Dt as useLayoutEffect,At as useMemo,Nt as useReducer,Tt as useRef,tn as useSignal,en as useSignalEffect,Pt as useState};+function t(){throw new Error("Cycle detected")}function n(){if(u>1){u--;return}let t,n=!1;while(void 0!==_){let i=_;_=void 0;f++;while(void 0!==i){const _=i.o;i.o=void 0;i.f&=-3;if(!(8&i.f)&&a(i))try{i.c()}catch(e){if(!n){t=e;n=!0}}i=_}}f=0;u--;if(n)throw t}function e(t){if(u>0)return t();u++;try{return t()}finally{n()}}let i,_,o=0;function r(t){if(o>0)return t();const n=i;i=void 0;o++;try{return t()}finally{o--;i=n}}let u=0,f=0,l=0;function s(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function c(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}c.prototype.h=function(){return!0};c.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};c.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};c.prototype.subscribe=function(t){const n=this;return S((function(){const e=n.value,i=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=i}}))};c.prototype.valueOf=function(){return this.value};c.prototype.toString=function(){return this.value+""};c.prototype.toJSON=function(){return this.value};c.prototype.peek=function(){return this.v};Object.defineProperty(c.prototype,"value",{get(){const t=s(this);if(void 0!==t)t.i=this.i;return this.v},set(e){if(i instanceof v)!function(){throw new Error("Computed cannot have side-effects")}();if(e!==this.v){if(f>100)t();this.v=e;this.i++;l++;u++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{n()}}}});function h(t){return new c(t)}function a(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function p(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function d(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function v(t){c.call(this,void 0);this.x=t;this.s=void 0;this.g=l-1;this.f=4}(v.prototype=new c).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===l)return!0;this.g=l;this.f|=1;if(this.i>0&&!a(this)){this.f&=-2;return!0}const t=i;try{p(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;d(this);this.f&=-2;return!0};v.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}c.prototype.S.call(this,t)};v.prototype.U=function(t){if(void 0!==this.t){c.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};v.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};v.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(v.prototype,"value",{get(){if(1&this.f)t();const n=s(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function y(t){return new v(t)}function m(t){const e=t.u;t.u=void 0;if("function"==typeof e){u++;const _=i;i=void 0;try{e()}catch(n){t.f&=-2;t.f|=8;g(t);throw n}finally{i=_;n()}}}function g(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;m(t)}function b(t){if(i!==this)throw new Error("Out-of-order effect");d(this);i=t;this.f&=-2;if(8&this.f)g(this);n()}function k(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}k.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};k.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;m(this);p(this);u++;const n=i;i=this;return b.bind(this,n)};k.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=_;_=this}};k.prototype.d=function(){this.f|=8;if(!(1&this.f))g(this)};function S(t){const n=new k(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,w,C,E,U,H,N,P,$,D={},T=[],V=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,A=Array.isArray;function F(t,n){for(var e in n)t[e]=n[e];return t}function M(t){var n=t.parentNode;n&&n.removeChild(t)}function W(t,n,e){var i,_,o,r={};for(o in n)"key"==o?i=n[o]:"ref"==o?_=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,i,_,null)}function O(t,n,e,i,_){var o={type:t,props:n,key:e,ref:i,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,__h:null,constructor:void 0,__v:null==_?++C:_};return null==_&&null!=w.vnode&&w.vnode(o),o}function L(){return{current:null}}function R(t){return t.children}function I(t,n){this.props=t,this.context=n}function j(t,n){if(null==n)return t.__?j(t.__,t.__.__k.indexOf(t)+1):null;for(var e;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e)return e.__e;return"function"==typeof t.type?j(t):null}function B(t){var n,e;if(null!=(t=t.__)&&null!=t.__c){for(t.__e=t.__c.base=null,n=0;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e){t.__e=t.__c.base=e.__e;break}return B(t)}}function q(t){(!t.__d&&(t.__d=!0)&&U.push(t)&&!G.__r++||H!==w.debounceRendering)&&((H=w.debounceRendering)||N)(G)}function G(){var t,n,e,i,_,o,r,u,f;for(U.sort(P);t=U.shift();)t.__d&&(n=U.length,i=void 0,_=void 0,o=void 0,u=(r=(e=t).__v).__e,(f=e.__P)&&(i=[],_=[],(o=F({},r)).__v=r.__v+1,it(f,r,o,e.__n,void 0!==f.ownerSVGElement,null!=r.__h?[u]:null,i,null==u?j(r):u,r.__h,_),_t(i,r,_),r.__e!=u&&B(r)),U.length>n&&U.sort(P));G.__r=0}function z(t,n,e,i,_,o,r,u,f,l,s){var c,h,a,p,d,v,y,m,g,b,k=0,S=i&&i.__k||T,x=S.length,w=x,C=n.length;for(e.__k=[],c=0;c<C;c++)null!=(p=e.__k[c]=null==(p=n[c])||"boolean"==typeof p||"function"==typeof p?null:"string"==typeof p||"number"==typeof p||"bigint"==typeof p?O(null,p,null,null,p):A(p)?O(R,{children:p},null,null,null):p.__b>0?O(p.type,p.props,p.key,p.ref?p.ref:null,p.__v):p)&&(p.__=e,p.__b=e.__b+1,-1===(m=X(p,S,y=c+k,w))?a=D:(a=S[m]||D,S[m]=void 0,w--),it(t,p,a,_,o,r,u,f,l,s),d=p.__e,(h=p.ref)&&a.ref!=h&&(a.ref&&rt(a.ref,null,p),s.push(h,p.__c||d,p)),null!=d&&(null==v&&(v=d),b=!(g=a===D||null===a.__v)&&m===y,g?-1==m&&k--:m!==y&&(m===y+1?(k++,b=!0):m>y?w>C-y?(k+=m-y,b=!0):k--:k=m<y&&m==y-1?m-y:0),y=c+k,b=b||m==c&&!g,"function"!=typeof p.type||m===y&&a.__k!==p.__k?"function"==typeof p.type||b?void 0!==p.__d?(f=p.__d,p.__d=void 0):f=d.nextSibling:f=Q(t,d,f):f=J(p,f,t),"function"==typeof e.type&&(e.__d=f)));for(e.__e=v,c=x;c--;)null!=S[c]&&("function"==typeof e.type&&null!=S[c].__e&&S[c].__e==e.__d&&(e.__d=S[c].__e.nextSibling),ut(S[c],S[c]))}function J(t,n,e){for(var i,_=t.__k,o=0;_&&o<_.length;o++)(i=_[o])&&(i.__=t,n="function"==typeof i.type?J(i,n,e):Q(e,i.__e,n));return n}function K(t,n){return n=n||[],null==t||"boolean"==typeof t||(A(t)?t.some((function(t){K(t,n)})):n.push(t)),n}function Q(t,n,e){return null==e||e.parentNode!==t?t.insertBefore(n,null):n==e&&null!=n.parentNode||t.insertBefore(n,e),n.nextSibling}function X(t,n,e,i){var _=t.key,o=t.type,r=e-1,u=e+1,f=n[e];if(null===f||f&&_==f.key&&o===f.type)return e;if(i>(null!=f?1:0))for(;r>=0||u<n.length;){if(r>=0){if((f=n[r])&&_==f.key&&o===f.type)return r;r--}if(u<n.length){if((f=n[u])&&_==f.key&&o===f.type)return u;u++}}return-1}function Y(t,n,e,i,_){var o;for(o in e)"children"===o||"key"===o||o in n||tt(t,o,null,e[o],i);for(o in n)_&&"function"!=typeof n[o]||"children"===o||"key"===o||"value"===o||"checked"===o||e[o]===n[o]||tt(t,o,n[o],e[o],i)}function Z(t,n,e){"-"===n[0]?t.setProperty(n,null==e?"":e):t[n]=null==e?"":"number"!=typeof e||V.test(n)?e:e+"px"}function tt(t,n,e,i,_){var o;t:if("style"===n)if("string"==typeof e)t.style.cssText=e;else{if("string"==typeof i&&(t.style.cssText=i=""),i)for(n in i)e&&n in e||Z(t.style,n,"");if(e)for(n in e)i&&e[n]===i[n]||Z(t.style,n,e[n])}else if("o"===n[0]&&"n"===n[1])o=n!==(n=n.replace(/Capture$/,"")),n=n.toLowerCase()in t?n.toLowerCase().slice(2):n.slice(2),t.l||(t.l={}),t.l[n+o]=e,e?i||t.addEventListener(n,o?et:nt,o):t.removeEventListener(n,o?et:nt,o);else if("dangerouslySetInnerHTML"!==n){if(_)n=n.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if("width"!==n&&"height"!==n&&"href"!==n&&"list"!==n&&"form"!==n&&"tabIndex"!==n&&"download"!==n&&"rowSpan"!==n&&"colSpan"!==n&&n in t)try{t[n]=null==e?"":e;break t}catch(t){}"function"==typeof e||(null==e||!1===e&&"-"!==n[4]?t.removeAttribute(n):t.setAttribute(n,e))}}function nt(t){return this.l[t.type+!1](w.event?w.event(t):t)}function et(t){return this.l[t.type+!0](w.event?w.event(t):t)}function it(t,n,e,i,_,o,r,u,f,l){var s,c,h,a,p,d,v,y,m,g,b,k,S,x,C,E=n.type;if(void 0!==n.constructor)return null;null!=e.__h&&(f=e.__h,u=n.__e=e.__e,n.__h=null,o=[u]),(s=w.__b)&&s(n);try{t:if("function"==typeof E){if(y=n.props,m=(s=E.contextType)&&i[s.__c],g=s?m?m.props.value:s.__:i,e.__c?v=(c=n.__c=e.__c).__=c.__E:("prototype"in E&&E.prototype.render?n.__c=c=new E(y,g):(n.__c=c=new I(y,g),c.constructor=E,c.render=ft),m&&m.sub(c),c.props=y,c.state||(c.state={}),c.context=g,c.__n=i,h=c.__d=!0,c.__h=[],c._sb=[]),null==c.__s&&(c.__s=c.state),null!=E.getDerivedStateFromProps&&(c.__s==c.state&&(c.__s=F({},c.__s)),F(c.__s,E.getDerivedStateFromProps(y,c.__s))),a=c.props,p=c.state,c.__v=n,h)null==E.getDerivedStateFromProps&&null!=c.componentWillMount&&c.componentWillMount(),null!=c.componentDidMount&&c.__h.push(c.componentDidMount);else{if(null==E.getDerivedStateFromProps&&y!==a&&null!=c.componentWillReceiveProps&&c.componentWillReceiveProps(y,g),!c.__e&&(null!=c.shouldComponentUpdate&&!1===c.shouldComponentUpdate(y,c.__s,g)||n.__v===e.__v)){for(n.__v!==e.__v&&(c.props=y,c.state=c.__s,c.__d=!1),n.__e=e.__e,n.__k=e.__k,n.__k.forEach((function(t){t&&(t.__=n)})),b=0;b<c._sb.length;b++)c.__h.push(c._sb[b]);c._sb=[],c.__h.length&&r.push(c);break t}null!=c.componentWillUpdate&&c.componentWillUpdate(y,c.__s,g),null!=c.componentDidUpdate&&c.__h.push((function(){c.componentDidUpdate(a,p,d)}))}if(c.context=g,c.props=y,c.__P=t,c.__e=!1,k=w.__r,S=0,"prototype"in E&&E.prototype.render){for(c.state=c.__s,c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),x=0;x<c._sb.length;x++)c.__h.push(c._sb[x]);c._sb=[]}else do{c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),c.state=c.__s}while(c.__d&&++S<25);c.state=c.__s,null!=c.getChildContext&&(i=F(F({},i),c.getChildContext())),h||null==c.getSnapshotBeforeUpdate||(d=c.getSnapshotBeforeUpdate(a,p)),z(t,A(C=null!=s&&s.type===R&&null==s.key?s.props.children:s)?C:[C],n,e,i,_,o,r,u,f,l),c.base=n.__e,n.__h=null,c.__h.length&&r.push(c),v&&(c.__E=c.__=null)}else null==o&&n.__v===e.__v?(n.__k=e.__k,n.__e=e.__e):n.__e=ot(e.__e,n,e,i,_,o,r,f,l);(s=w.diffed)&&s(n)}catch(t){n.__v=null,(f||null!=o)&&(n.__e=u,n.__h=!!f,o[o.indexOf(u)]=null),w.__e(t,n,e)}}function _t(t,n,e){for(var i=0;i<e.length;i++)rt(e[i],e[++i],e[++i]);w.__c&&w.__c(n,t),t.some((function(n){try{t=n.__h,n.__h=[],t.some((function(t){t.call(n)}))}catch(t){w.__e(t,n.__v)}}))}function ot(t,n,e,i,_,o,r,u,f){var l,s,c,h=e.props,a=n.props,p=n.type,d=0;if("svg"===p&&(_=!0),null!=o)for(;d<o.length;d++)if((l=o[d])&&"setAttribute"in l==!!p&&(p?l.localName===p:3===l.nodeType)){t=l,o[d]=null;break}if(null==t){if(null===p)return document.createTextNode(a);t=_?document.createElementNS("http://www.w3.org/2000/svg",p):document.createElement(p,a.is&&a),o=null,u=!1}if(null===p)h===a||u&&t.data===a||(t.data=a);else{if(o=o&&x.call(t.childNodes),s=(h=e.props||D).dangerouslySetInnerHTML,c=a.dangerouslySetInnerHTML,!u){if(null!=o)for(h={},d=0;d<t.attributes.length;d++)h[t.attributes[d].name]=t.attributes[d].value;(c||s)&&(c&&(s&&c.__html==s.__html||c.__html===t.innerHTML)||(t.innerHTML=c&&c.__html||""))}if(Y(t,a,h,_,u),c)n.__k=[];else if(z(t,A(d=n.props.children)?d:[d],n,e,i,_&&"foreignObject"!==p,o,r,o?o[0]:e.__k&&j(e,0),u,f),null!=o)for(d=o.length;d--;)null!=o[d]&&M(o[d]);u||("value"in a&&void 0!==(d=a.value)&&(d!==t.value||"progress"===p&&!d||"option"===p&&d!==h.value)&&tt(t,"value",d,h.value,!1),"checked"in a&&void 0!==(d=a.checked)&&d!==t.checked&&tt(t,"checked",d,h.checked,!1))}return t}function rt(t,n,e){try{"function"==typeof t?t(n):t.current=n}catch(t){w.__e(t,e)}}function ut(t,n,e){var i,_;if(w.unmount&&w.unmount(t),(i=t.ref)&&(i.current&&i.current!==t.__e||rt(i,null,n)),null!=(i=t.__c)){if(i.componentWillUnmount)try{i.componentWillUnmount()}catch(t){w.__e(t,n)}i.base=i.__P=null,t.__c=void 0}if(i=t.__k)for(_=0;_<i.length;_++)i[_]&&ut(i[_],n,e||"function"!=typeof t.type);e||null==t.__e||M(t.__e),t.__=t.__e=t.__d=void 0}function ft(t,n,e){return this.constructor(t,e)}function lt(t,n,e){var i,_,o,r;w.__&&w.__(t,n),_=(i="function"==typeof e)?null:e&&e.__k||n.__k,o=[],r=[],it(n,t=(!i&&e||n).__k=W(R,null,[t]),_||D,D,void 0!==n.ownerSVGElement,!i&&e?[e]:_?null:n.firstChild?x.call(n.childNodes):null,o,!i&&e?e:_?_.__e:n.firstChild,i,r),_t(o,t,r)}function st(t,n){lt(t,n,st)}function ct(t,n,e){var i,_,o,r,u=F({},t.props);for(o in t.type&&t.type.defaultProps&&(r=t.type.defaultProps),n)"key"==o?i=n[o]:"ref"==o?_=n[o]:u[o]=void 0===n[o]&&void 0!==r?r[o]:n[o];return arguments.length>2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,i||t.key,_||t.ref,null)}function ht(t,n){var e={__c:n="__cC"+$++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,i;return this.getChildContext||(e=[],(i={})[n]=this,this.getChildContext=function(){return i},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,q(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=T.slice,w={__e:function(t,n,e,i){for(var _,o,r;n=n.__;)if((_=n.__c)&&!_.__)try{if((o=_.constructor)&&null!=o.getDerivedStateFromError&&(_.setState(o.getDerivedStateFromError(t)),r=_.__d),null!=_.componentDidCatch&&(_.componentDidCatch(t,i||{}),r=_.__d),r)return _.__E=_}catch(n){t=n}throw t}},C=0,E=function(t){return null!=t&&void 0===t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=F({},this.state),"function"==typeof t&&(t=t(F({},e),this.props)),t&&F(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),q(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),q(this))},I.prototype.render=R,U=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,P=function(t,n){return t.__v.__b-n.__v.__b},G.__r=0,$=0;var at,pt,dt,vt,yt=0,mt=[],gt=[],bt=w.__b,kt=w.__r,St=w.diffed,xt=w.__c,wt=w.unmount;function Ct(t,n){w.__h&&w.__h(pt,t,yt||n),yt=0;var e=pt.__H||(pt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:gt}),e.__[t]}function Et(t){return yt=1,Ut(Bt,t)}function Ut(t,n,e){var i=Ct(at++,2);if(i.t=t,!i.__c&&(i.__=[e?e(n):Bt(void 0,n),function(t){var n=i.__N?i.__N[0]:i.__[0],e=i.t(n,t);n!==e&&(i.__N=[e,i.__[1]],i.__c.setState({}))}],i.__c=pt,!pt.u)){var _=function(t,n,e){if(!i.__c.__H)return!0;var _=i.__c.__H.__.filter((function(t){return t.__c}));if(_.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return _.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&i.__c.props===t)&&(!o||o.call(this,t,n,e))};pt.u=!0;var o=pt.shouldComponentUpdate,r=pt.componentWillUpdate;pt.componentWillUpdate=function(t,n,e){if(this.__e){var i=o;o=void 0,_(t,n,e),o=i}r&&r.call(this,t,n,e)},pt.shouldComponentUpdate=_}return i.__N||i.__}function Ht(t,n){var e=Ct(at++,3);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__H.__h.push(e))}function Nt(t,n){var e=Ct(at++,4);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__h.push(e))}function Pt(t){return yt=5,Dt((function(){return{current:t}}),[])}function $t(t,n,e){yt=6,Nt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function Dt(t,n){var e=Ct(at++,7);return jt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Tt(t,n){return yt=8,Dt((function(){return t}),n)}function Vt(t){var n=pt.context[t.__c],e=Ct(at++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(pt)),n.props.value):t.__}function At(t,n){w.useDebugValue&&w.useDebugValue(n?n(t):t)}function Ft(t){var n=Ct(at++,10),e=Et();return n.__=t,pt.componentDidCatch||(pt.componentDidCatch=function(t,i){n.__&&n.__(t,i),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Mt(){var t=Ct(at++,11);if(!t.__){for(var n=pt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Wt(){for(var t;t=mt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(Rt),t.__H.__h.forEach(It),t.__H.__h=[]}catch(u){t.__H.__h=[],w.__e(u,t.__v)}}w.__b=function(t){pt=null,bt&&bt(t)},w.__r=function(t){kt&&kt(t),at=0;var n=(pt=t.__c).__H;n&&(dt===pt?(n.__h=[],pt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=gt,t.__N=t.i=void 0}))):(n.__h.forEach(Rt),n.__h.forEach(It),n.__h=[],at=0)),dt=pt},w.diffed=function(t){St&&St(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==mt.push(n)&&vt===w.requestAnimationFrame||((vt=w.requestAnimationFrame)||Lt)(Wt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==gt&&(t.__=t.__V),t.i=void 0,t.__V=gt}))),dt=pt=null},w.__c=function(t,n){n.some((function(t){try{t.__h.forEach(Rt),t.__h=t.__h.filter((function(t){return!t.__||It(t)}))}catch(s){n.some((function(t){t.__h&&(t.__h=[])})),n=[],w.__e(s,t.__v)}})),xt&&xt(t,n)},w.unmount=function(t){wt&&wt(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{Rt(t)}catch(t){n=t}})),e.__H=void 0,n&&w.__e(n,e.__v))};var Ot="function"==typeof requestAnimationFrame;function Lt(t){var n,e=function(){clearTimeout(i),Ot&&cancelAnimationFrame(n),setTimeout(t)},i=setTimeout(e,100);Ot&&(n=requestAnimationFrame(e))}function Rt(t){var n=pt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),pt=n}function It(t){var n=pt;t.__c=t.__(),pt=n}function jt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function Bt(t,n){return"function"==typeof n?n(t):n}function qt(t,n){w[t]=n.bind(null,w[t]||(()=>{}))}let Gt,zt;function Jt(t){if(zt)zt();zt=t&&t.S()}function Kt({data:t}){const n=Xt(t);n.value=t;const e=Dt(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!E(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return y(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Kt.displayName="_st";Object.defineProperties(c.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Kt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});qt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let i in e){if("children"===i)continue;let _=e[i];if(_ instanceof c){if(!t)n.__np=t={};t[i]=_;e[i]=_.peek()}}}t(n)});qt("__r",(t,n)=>{Jt();let e,i=n.__c;if(i){i.__$f&=-2;e=i.__$u;if(void 0===e)i.__$u=e=function(t){let n;S((function(){n=this}));n.c=()=>{i.__$f|=1;i.setState({})};return n}()}Gt=i;Jt(e);t(n)});qt("__e",(t,n,e,i)=>{Jt();Gt=void 0;t(n,e,i)});qt("diffed",(t,n)=>{Jt();Gt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,i=n.props;if(t){let n=e.U;if(n)for(let e in n){let i=n[e];if(void 0!==i&&!(e in t)){i.d();n[e]=void 0}}else{n={};e.U=n}for(let _ in t){let o=n[_],r=t[_];if(void 0===o){o=Qt(e,_,r,i);n[_]=o}else o.o(r,i)}}}t(n)});function Qt(t,n,e,i){const _=n in t&&void 0===t.ownerSVGElement,o=h(e);return{o:(t,n)=>{o.value=t;i=n},d:S(()=>{const e=o.value.value;if(i[n]!==e){i[n]=e;if(_)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}qt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});qt("__h",(t,n,e,i)=>{if(i<3||9===i)n.__$f|=2;t(n,e,i)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let i in n)return!0;for(let i in t)if("__source"!==i&&t[i]!==this.props[i])return!0;for(let i in this.props)if(!(i in t))return!0;return!1};function Xt(t){return Dt(()=>h(t),[])}function Yt(t){const n=Pt(t);n.current=t;Gt.__$f|=4;return Dt(()=>y(()=>n.current()),[])}function Zt(t){const n=Pt(t);n.current=t;Ht(()=>S(()=>n.current()),[])}var tn=function(t,n,e,i){var _;n[0]=0;for(var o=1;o<n.length;o++){var r=n[o++],u=n[o]?(n[0]|=r?1:2,e[n[o++]]):n[++o];3===r?i[0]=u:4===r?i[1]=Object.assign(i[1]||{},u):5===r?(i[1]=i[1]||{})[n[++o]]=u:6===r?i[1][n[++o]]+=u+"":r?(_=t.apply(u,tn(t,u,e,["",null])),i.push(_),u[0]?n[0]|=2:(n[o-2]=0,n[o]=_)):i.push(u)}return i},nn=new Map;function en(t){var n=nn.get(this);return n||(n=new Map,nn.set(this,n)),(n=tn(this,n.get(t)||(n.set(t,n=function(t){for(var n,e,i=1,_="",o="",r=[0],u=function(t){1===i&&(t||(_=_.replace(/^\s*\n\s*|\s*\n\s*$/g,"")))?r.push(0,t,_):3===i&&(t||_)?(r.push(3,t,_),i=2):2===i&&"..."===_&&t?r.push(4,t,0):2===i&&_&&!t?r.push(5,0,!0,_):i>=5&&((_||!t&&5===i)&&(r.push(i,0,_,e),i=6),t&&(r.push(i,t,0,e),i=6)),_=""},f=0;f<t.length;f++){f&&(1===i&&u(),u(f));for(var l=0;l<t[f].length;l++)n=t[f][l],1===i?"<"===n?(u(),r=[r],i=3):_+=n:4===i?"--"===_&&">"===n?(i=1,_=""):_=n+_[0]:o?n===o?o="":_+=n:'"'===n||"'"===n?o=n:">"===n?(u(),i=1):i&&("="===n?(i=5,e=_,_=""):"/"===n&&(i<5||">"===t[f][l+1])?(u(),3===i&&(r=r[0]),i=r,(r=r[0]).push(2,0,i),i=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),i=2):_+=n),3===i&&"!--"===_&&(i=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var _n=en.bind(W);export{I as Component,R as Fragment,c as Signal,e as batch,ct as cloneElement,y as computed,ht as createContext,W as createElement,L as createRef,S as effect,W as h,_n as html,st as hydrate,E as isValidElement,w as options,lt as render,h as signal,K as toChildArray,r as untracked,Tt as useCallback,Yt as useComputed,Vt as useContext,At as useDebugValue,Ht as useEffect,Ft as useErrorBoundary,Mt as useId,$t as useImperativeHandle,Nt as useLayoutEffect,Dt as useMemo,Ut as useReducer,Pt as useRef,Xt as useSignal,Zt as useSignalEffect,Et as useState};diff --git a/llama.cpp/server/public/json-schema-to-grammar.mjs b/llama.cpp/server/public/json-schema-to-grammar.mjsindex baa6761c..3f1b255c 100644--- a/llama.cpp/server/public/json-schema-to-grammar.mjs+++ b/llama.cpp/server/public/json-schema-to-grammar.mjs@@ -1,50 +1,25 @@-// WARNING: This file was ported from json-schema-to-grammar.py, please fix bugs / add features there first.const SPACE_RULE = '" "?';const PRIMITIVE_RULES = {boolean: '("true" | "false") space',number: '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space',integer: '("-"? ([0-9] | [1-9] [0-9]*)) space',- value: 'object | array | string | number | boolean',- object: '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space',- array: '"[" space ( value ("," space value)* )? "]" space',- uuid: '"\\"" ' + [8, 4, 4, 4, 12].map(n => [...new Array(n)].map(_ => '[0-9a-fA-F]').join('')).join(' "-" ') + ' "\\"" space',string: ` "\\"" ([^"\\\\] |"\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]))* "\\"" space`,null: '"null" space',};-const OBJECT_RULE_NAMES = ['object', 'array', 'string', 'number', 'boolean', 'null', 'value'];--// TODO: support "uri", "email" string formats-const DATE_RULES = {- 'date' : '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )',- 'time' : '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )',- 'date-time': 'date "T" time',- 'date-string': '"\\"" date "\\"" space',- 'time-string': '"\\"" time "\\"" space',- 'date-time-string': '"\\"" date-time "\\"" space',-};--const RESERVED_NAMES = {'root': true, ...PRIMITIVE_RULES, ...DATE_RULES};const INVALID_RULE_CHARS_RE = /[^\dA-Za-z-]+/g;const GRAMMAR_LITERAL_ESCAPE_RE = /[\n\r"]/g;-const GRAMMAR_RANGE_LITERAL_ESCAPE_RE = /[\n\r"\]\-\\]/g;-const GRAMMAR_LITERAL_ESCAPES = { '\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]' };--const NON_LITERAL_SET = new Set('|.()[]{}*+?');-const ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = new Set('[]()|{}*+?');+const GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"'};export class SchemaConverter {- constructor(options) {- this._propOrder = options.prop_order || {};- this._allowFetch = options.allow_fetch || false;- this._dotall = options.dotall || false;- this._rules = {'space': SPACE_RULE};- this._refs = {};- this._refsBeingResolved = new Set();+ constructor(propOrder) {+ this._propOrder = propOrder || {};+ this._rules = new Map();+ this._rules.set('space', SPACE_RULE);}_formatLiteral(literal) {@@ -55,490 +30,83 @@ export class SchemaConverter {return `"${escaped}"`;}- _formatRangeChar(literal) {- return JSON.stringify(literal).slice(1, -1).replace(- GRAMMAR_RANGE_LITERAL_ESCAPE_RE,- m => GRAMMAR_LITERAL_ESCAPES[m]- );- }-_addRule(name, rule) {let escName = name.replace(INVALID_RULE_CHARS_RE, '-');let key = escName;- if (escName in this._rules) {- if (this._rules[escName] === rule) {+ if (this._rules.has(escName)) {+ if (this._rules.get(escName) === rule) {return key;}let i = 0;- while ((`${escName}${i}` in this._rules) && (this._rules[`${escName}${i}`] !== rule)) {+ while (this._rules.has(`${escName}${i}`)) {i += 1;}key = `${escName}${i}`;}- this._rules[key] = rule;+ this._rules.set(key, rule);return key;}- async resolveRefs(schema, url) {- const visit = async (n) => {- if (Array.isArray(n)) {- return Promise.all(n.map(visit));- } else if (typeof n === 'object' && n !== null) {- let ref = n.$ref;- let target;- if (ref !== undefined && !this._refs[ref]) {- if (ref.startsWith('https://')) {- if (!this._allowFetch) {- throw new Error('Fetching remote schemas is not allowed (use --allow-fetch for force)');- }- const fetch = (await import('node-fetch')).default;-- const fragSplit = ref.split('#');- const baseUrl = fragSplit[0];-- target = this._refs[baseUrl];- if (!target) {- target = await this.resolveRefs(await fetch(ref).then(res => res.json()), baseUrl);- this._refs[baseUrl] = target;- }-- if (fragSplit.length === 1 || fragSplit[fragSplit.length - 1] === '') {- return target;- }- } else if (ref.startsWith('#/')) {- target = schema;- ref = `${url}${ref}`;- n.$ref = ref;- } else {- throw new Error(`Unsupported ref ${ref}`);- }-- const selectors = ref.split('#')[1].split('/').slice(1);- for (const sel of selectors) {- if (!target || !(sel in target)) {- throw new Error(`Error resolving ref ${ref}: ${sel} not in ${JSON.stringify(target)}`);- }- target = target[sel];- }-- this._refs[ref] = target;- } else {- await Promise.all(Object.values(n).map(visit));- }- }-- return n;- };-- return visit(schema);- }-- _generateUnionRule(name, altSchemas) {- return altSchemas- .map((altSchema, i) => this.visit(altSchema, `${name ?? ''}${name ? '-' : 'alternative-'}${i}`))- .join(' | ');- }-- _visitPattern(pattern, name) {- if (!pattern.startsWith('^') || !pattern.endsWith('$')) {- throw new Error('Pattern must start with "^" and end with "$"');- }- pattern = pattern.slice(1, -1);- const subRuleIds = {};-- let i = 0;- const length = pattern.length;-- const getDot = () => {- let rule;- if (this._dotall) {- rule = '[\\U00000000-\\U0010FFFF]';- } else {- // Accept any character... except \n and \r line break chars (\x0A and \xOD)- rule = '[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]';- }- return this._addRule('dot', rule);- };--- const toRule = ([s, isLiteral]) => isLiteral ? "\"" + s + "\"" : s;-- const transform = () => {- const start = i;- // For each component of this sequence, store its string representation and whether it's a literal.- // We only need a flat structure here to apply repetition operators to the last item, and- // to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially- // (GBNF's syntax is luckily very close to regular expressions!)- const seq = [];-- const joinSeq = () => {- const ret = [];- for (const [isLiteral, g] of groupBy(seq, x => x[1])) {- if (isLiteral) {- ret.push([[...g].map(x => x[0]).join(''), true]);- } else {- ret.push(...g);- }- }- if (ret.length === 1) {- return ret[0];- }- return [ret.map(x => toRule(x)).join(' '), false];- };-- while (i < length) {- const c = pattern[i];- if (c === '.') {- seq.push([getDot(), false]);- i += 1;- } else if (c === '(') {- i += 1;- if (i < length) {- if (pattern[i] === '?') {- throw new Error(`Unsupported pattern syntax "${pattern[i]}" at index ${i} of /${pattern}/`);- }- }- seq.push([`(${toRule(transform())})`, false]);- } else if (c === ')') {- i += 1;- if (start <= 0 || pattern[start - 1] !== '(') {- throw new Error(`Unbalanced parentheses; start = ${start}, i = ${i}, pattern = ${pattern}`);- }- return joinSeq();- } else if (c === '[') {- let squareBrackets = c;- i += 1;- while (i < length && pattern[i] !== ']') {- if (pattern[i] === '\\') {- squareBrackets += pattern.slice(i, i + 2);- i += 2;- } else {- squareBrackets += pattern[i];- i += 1;- }- }- if (i >= length) {- throw new Error(`Unbalanced square brackets; start = ${start}, i = ${i}, pattern = ${pattern}`);- }- squareBrackets += ']';- i += 1;- seq.push([squareBrackets, false]);- } else if (c === '|') {- seq.push(['|', false]);- i += 1;- } else if (c === '*' || c === '+' || c === '?') {- seq[seq.length - 1] = [toRule(seq[seq.length - 1]) + c, false];- i += 1;- } else if (c === '{') {- let curlyBrackets = c;- i += 1;- while (i < length && pattern[i] !== '}') {- curlyBrackets += pattern[i];- i += 1;- }- if (i >= length) {- throw new Error(`Unbalanced curly brackets; start = ${start}, i = ${i}, pattern = ${pattern}`);- }- curlyBrackets += '}';- i += 1;- const nums = curlyBrackets.slice(1, -1).split(',').map(s => s.trim());- let minTimes, maxTimes;- if (nums.length === 1) {- minTimes = parseInt(nums[0], 10);- maxTimes = minTimes;- } else {- if (nums.length !== 2) {- throw new Error(`Invalid quantifier ${curlyBrackets}`);- }- minTimes = nums[0] ? parseInt(nums[0], 10) : 0;- maxTimes = nums[1] ? parseInt(nums[1], 10) : Infinity;- }-- let [sub, subIsLiteral] = seq[seq.length - 1];-- if (minTimes === 0 && maxTimes === Infinity) {- seq[seq.length - 1] = [`${sub}*`, false];- } else if (minTimes === 0 && maxTimes === 1) {- seq[seq.length - 1] = [`${sub}?`, false];- } else if (minTimes === 1 && maxTimes === Infinity) {- seq[seq.length - 1] = [`${sub}+`, false];- } else {- if (!subIsLiteral) {- let id = subRuleIds[sub];- if (id === undefined) {- id = this._addRule(`${name}-${Object.keys(subRuleIds).length + 1}`, sub);- subRuleIds[sub] = id;- }- sub = id;- }-- const repeatedSub = Array.from({ length: minTimes }, () => subIsLiteral ? `"${sub.slice(1, -1).repeat(minTimes)}"` : sub);- const optionalSub = maxTimes !== undefined ? Array.from({ length: maxTimes - minTimes }, () => `${sub}?`) : [`${sub}*`];- seq[seq.length - 1] = [repeatedSub.concat(optionalSub).join(' '), false];- }- } else {- let literal = '';- while (i < length) {- if (pattern[i] === '\\' && i < length - 1) {- const next = pattern[i + 1];- if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.has(next)) {- i += 1;- literal += pattern[i];- i += 1;- } else {- literal += pattern.slice(i, i + 2);- i += 2;- }- } else if (pattern[i] === '"') {- literal += '\\"';- i += 1;- } else if (!NON_LITERAL_SET.has(pattern[i]) &&- (i === length - 1 || literal === '' || pattern[i + 1] === '.' || !NON_LITERAL_SET.has(pattern[i+1]))) {- literal += pattern[i];- i += 1;- } else {- break;- }- }- if (literal !== '') {- seq.push([literal, true]);- }- }- }-- return joinSeq();- };-- return this._addRule(name, "\"\\\"\" " + toRule(transform()) + " \"\\\"\" space")- }-- _resolveRef(ref) {- let refName = ref.split('/').pop();- if (!(refName in this._rules) && !this._refsBeingResolved.has(ref)) {- this._refsBeingResolved.add(ref);- const resolved = this._refs[ref];- refName = this.visit(resolved, refName);- this._refsBeingResolved.delete(ref);- }- return refName;- }-- _generateConstantRule(value) {- if (typeof value !== 'string') {- throw new Error('Only string constants are supported, got ' + JSON.stringify(value));- }- return this._formatLiteral(value);- }-visit(schema, name) {const schemaType = schema.type;- const schemaFormat = schema.format;- const ruleName = name in RESERVED_NAMES ? name + '-' : name == '' ? 'root' : name;+ const ruleName = name || 'root';++ if (schema.oneOf || schema.anyOf) {+ const rule = (schema.oneOf || schema.anyOf).map((altSchema, i) =>+ this.visit(altSchema, `${name}${name ? "-" : ""}${i}`)+ ).join(' | ');- const ref = schema.$ref;- if (ref !== undefined) {- return this._addRule(ruleName, this._resolveRef(ref));- } else if (schema.oneOf || schema.anyOf) {- return this._addRule(ruleName, this._generateUnionRule(name, schema.oneOf || schema.anyOf));- } else if (Array.isArray(schemaType)) {- return this._addRule(ruleName, this._generateUnionRule(name, schemaType.map(t => ({ type: t }))));+ return this._addRule(ruleName, rule);} else if ('const' in schema) {- if (typeof schema.const !== 'string') {- throw new Error('Only string constants are supported, got ' + JSON.stringify(schema.const));- }- return this._addRule(ruleName, this._generateConstantRule(schema.const));+ return this._addRule(ruleName, this._formatLiteral(schema.const));} else if ('enum' in schema) {- const rule = schema.enum.map(v => this._generateConstantRule(v)).join(' | ');+ const rule = schema.enum.map(v => this._formatLiteral(v)).join(' | ');return this._addRule(ruleName, rule);- } else if ((schemaType === undefined || schemaType === 'object') &&- ('properties' in schema ||- ('additionalProperties' in schema && schema.additionalProperties !== true))) {- const required = new Set(schema.required || []);- const properties = Object.entries(schema.properties ?? {});- return this._addRule(ruleName, this._buildObjectRule(properties, required, name, schema.additionalProperties));- } else if ((schemaType === undefined || schemaType === 'object') && 'allOf' in schema) {- const required = new Set();- const properties = [];- const addComponent = (compSchema, isRequired) => {- const ref = compSchema.$ref;- if (ref !== undefined) {- compSchema = this._refs[ref];+ } else if (schemaType === 'object' && 'properties' in schema) {+ // TODO: `required` keyword (from python implementation)+ const propOrder = this._propOrder;+ const propPairs = Object.entries(schema.properties).sort((a, b) => {+ // sort by position in prop_order (if specified) then by key+ const orderA = typeof propOrder[a[0]] === 'number' ? propOrder[a[0]] : Infinity;+ const orderB = typeof propOrder[b[0]] === 'number' ? propOrder[b[0]] : Infinity;+ return orderA - orderB || a[0].localeCompare(b[0]);+ });++ let rule = '"{" space';+ propPairs.forEach(([propName, propSchema], i) => {+ const propRuleName = this.visit(propSchema, `${name}${name ? "-" : ""}${propName}`);+ if (i > 0) {+ rule += ' "," space';}+ rule += ` ${this._formatLiteral(propName)} space ":" space ${propRuleName}`;+ });+ rule += ' "}" space';- if ('properties' in compSchema) {- for (const [propName, propSchema] of Object.entries(compSchema.properties)) {- properties.push([propName, propSchema]);- if (isRequired) {- required.add(propName);- }- }- }- };-- for (const t of schema.allOf) {- if ('anyOf' in t) {- for (const tt of t.anyOf) {- addComponent(tt, false);- }- } else {- addComponent(t, true);- }- }-- return this._addRule(ruleName, this._buildObjectRule(properties, required, name, /* additionalProperties= */ false));- } else if ((schemaType === undefined || schemaType === 'array') && ('items' in schema || 'prefixItems' in schema)) {- const items = schema.items ?? schema.prefixItems;- if (Array.isArray(items)) {- return this._addRule(- ruleName,- '"[" space ' +- items.map((item, i) => this.visit(item, `${name ?? ''}${name ? '-' : ''}tuple-${i}`)).join(' "," space ') +- ' "]" space'- );- } else {- const itemRuleName = this.visit(items, `${name ?? ''}${name ? '-' : ''}item`);- const listItemOperator = `( "," space ${itemRuleName} )`;- let successiveItems = '';- let minItems = schema.minItems || 0;- const maxItems = schema.maxItems;- if (minItems > 0) {- successiveItems = listItemOperator.repeat(minItems - 1);- minItems--;- }- if (maxItems !== undefined && maxItems > minItems) {- successiveItems += `${listItemOperator}?`.repeat(maxItems - minItems - 1);- } else {- successiveItems += `${listItemOperator}*`;- }- const rule = minItems === 0- ? `"[" space ( ${itemRuleName} ${successiveItems} )? "]" space`- : `"[" space ${itemRuleName} ${successiveItems} "]" space`;- return this._addRule(ruleName, rule);- }- } else if ((schemaType === undefined || schemaType === 'string') && 'pattern' in schema) {- return this._visitPattern(schema.pattern, ruleName);- } else if ((schemaType === undefined || schemaType === 'string') && /^uuid[1-5]?$/.test(schema.format || '')) {- return this._addRule(- ruleName === 'root' ? 'root' : schemaFormat,- PRIMITIVE_RULES['uuid'])- } else if ((schemaType === undefined || schemaType === 'string') && schema.format in DATE_RULES) {- for (const [t, r] of Object.entries(DATE_RULES)) {- this._addRule(t, r);- }- return schemaFormat + '-string';- } else if ((schemaType === 'object') || (Object.keys(schema).length === 0)) {- for (const n of OBJECT_RULE_NAMES) {- this._addRule(n, PRIMITIVE_RULES[n]);- }- return this._addRule(ruleName, 'object');+ return this._addRule(ruleName, rule);+ } else if (schemaType === 'array' && 'items' in schema) {+ // TODO `prefixItems` keyword (from python implementation)+ const itemRuleName = this.visit(schema.items, `${name}${name ? "-" : ""}item`);+ const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`;+ return this._addRule(ruleName, rule);} else {- if (!(schemaType in PRIMITIVE_RULES)) {+ if (!PRIMITIVE_RULES[schemaType]) {throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`);}- // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero- return this._addRule(ruleName === 'root' ? 'root' : schemaType, PRIMITIVE_RULES[schemaType]);- }- }-- _buildObjectRule(properties, required, name, additionalProperties) {- const propOrder = this._propOrder;- // sort by position in prop_order (if specified) then by original order- const sortedProps = properties.map(([k]) => k).sort((a, b) => {- const orderA = propOrder[a] || Infinity;- const orderB = propOrder[b] || Infinity;- return orderA - orderB || properties.findIndex(([k]) => k === a) - properties.findIndex(([k]) => k === b);- });-- const propKvRuleNames = {};- for (const [propName, propSchema] of properties) {- const propRuleName = this.visit(propSchema, `${name ?? ''}${name ? '-' : ''}${propName}`);- propKvRuleNames[propName] = this._addRule(- `${name ?? ''}${name ? '-' : ''}${propName}-kv`,- `${this._formatLiteral(propName)} space ":" space ${propRuleName}`+ return this._addRule(+ ruleName === 'root' ? 'root' : schemaType,+ PRIMITIVE_RULES[schemaType]);}- const requiredProps = sortedProps.filter(k => required.has(k));- const optionalProps = sortedProps.filter(k => !required.has(k));-- if (typeof additionalProperties === 'object' || additionalProperties === true) {- const subName = `${name ?? ''}${name ? '-' : ''}additional`;- const valueRule = this.visit(additionalProperties === true ? {} : additionalProperties, `${subName}-value`);- propKvRuleNames['*'] = this._addRule(- `${subName}-kv`,- `${this._addRule('string', PRIMITIVE_RULES['string'])} ":" space ${valueRule}`);- optionalProps.push('*');- }-- let rule = '"{" space ';- rule += requiredProps.map(k => propKvRuleNames[k]).join(' "," space ');-- if (optionalProps.length > 0) {- rule += ' (';- if (requiredProps.length > 0) {- rule += ' "," space ( ';- }-- const getRecursiveRefs = (ks, firstIsOptional) => {- const [k, ...rest] = ks;- const kvRuleName = propKvRuleNames[k];- let res;- if (k === '*') {- res = this._addRule(- `${name ?? ''}${name ? '-' : ''}additional-kvs`,- `${kvRuleName} ( "," space ` + kvRuleName + ` )*`- )- } else if (firstIsOptional) {- res = `( "," space ${kvRuleName} )?`;- } else {- res = kvRuleName;- }- if (rest.length > 0) {- res += ' ' + this._addRule(- `${name ?? ''}${name ? '-' : ''}${k}-rest`,- getRecursiveRefs(rest, true)- );- }- return res;- };-- rule += optionalProps.map((_, i) => getRecursiveRefs(optionalProps.slice(i), false)).join(' | ');- if (requiredProps.length > 0) {- rule += ' )';- }- rule += ' )?';- }-- rule += ' "}" space';-- return rule;}formatGrammar() {let grammar = '';- for (const [name, rule] of Object.entries(this._rules).sort(([a], [b]) => a.localeCompare(b))) {+ this._rules.forEach((rule, name) => {grammar += `${name} ::= ${rule}\n`;- }+ });return grammar;}}--// Helper function to group elements by a key function-function* groupBy(iterable, keyFn) {- let lastKey = null;- let group = [];- for (const element of iterable) {- const key = keyFn(element);- if (lastKey !== null && key !== lastKey) {- yield [lastKey, group];- group = [];- }- group.push(element);- lastKey = key;- }- if (group.length > 0) {- yield [lastKey, group];- }-}diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 914482ab..4d3a3e27 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -1,141 +1,152 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#include "utils.h"-#include "macsandbox.h"-#include "llama.cpp/common.h"#include "llama.cpp/llama.h"#include "llama.cpp/grammar-parser.h"-#include "llama.cpp/json-schema-to-grammar.h"+#include "llama.cpp/llava/llava.h"+#include "llama.cpp/stb_image.h"+#include "utils.h"+#include "oai.h"+#include "macsandbox.h"// increase max payload length to allow use of larger context size#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576#include "httplib.h"#include "llama.cpp/json.h"-#include <atomic>-#include <chrono>-#include <condition_variable>#include <cstddef>-#include <set>-#include <mutex>#include <thread>+#include <chrono>+#include <condition_variable>+#include <atomic>#include <signal.h>-#include <memory>#include <libc/calls/pledge.h>#include <tool/args/args.h>#include <libc/dce.h>using json = nlohmann::json;-bool server_verbose = false;-bool server_log_json = true;--enum stop_type {- STOP_TYPE_FULL,- STOP_TYPE_PARTIAL,-};--enum slot_state {- SLOT_STATE_IDLE,- SLOT_STATE_PROCESSING,-};--enum slot_command {- SLOT_COMMAND_NONE,- SLOT_COMMAND_LOAD_PROMPT,- SLOT_COMMAND_RELEASE,-};--enum server_state {- SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet- SERVER_STATE_READY, // Server is ready and model is loaded- SERVER_STATE_ERROR // An error occurred, load_model failed-};--enum server_task_type {- SERVER_TASK_TYPE_COMPLETION,- SERVER_TASK_TYPE_CANCEL,- SERVER_TASK_TYPE_NEXT_RESPONSE,- SERVER_TASK_TYPE_METRICS-};--struct server_task {- int id = -1; // to be filled by server_queue- int id_multi = -1;- int id_target = -1;-- server_task_type type;- json data;-- bool infill = false;- bool embedding = false;-};--struct server_task_result {- int id = -1;- int id_multi = -1;-- json data;-- bool stop;- bool error;-};--struct server_task_multi {- int id = -1;-- std::set<int> subtasks_remaining;- std::vector<server_task_result> results;+struct server_params+{+ std::string hostname = "127.0.0.1";+ std::vector<std::string> api_keys;+ std::string public_path = "/zip/llama.cpp/server/public";+ std::string chat_template = "";+ int32_t port = 8080;+ int32_t read_timeout = 600;+ int32_t write_timeout = 600;+ bool nobrowser = false;+ bool slots_endpoint = true;+ bool metrics_endpoint = false;};-struct slot_params {- bool stream = true;- bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt-- uint32_t seed = -1; // RNG seed- int32_t n_keep = 0; // number of tokens to keep from initial prompt- int32_t n_predict = -1; // new tokens to predict+bool server_verbose = false;+bool server_log_json = true;- std::vector<std::string> antiprompt;+static size_t common_part(const std::vector<llama_token> &a, const std::vector<llama_token> &b)+{+ size_t i;+ for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++)+ {+ }+ return i;+}- json input_prefix;- json input_suffix;+enum stop_type+{+ STOP_FULL,+ STOP_PARTIAL,};-struct server_params {- int32_t port = 8080;- int32_t read_timeout = 600;- int32_t write_timeout = 600;- int32_t n_threads_http = -1;-- std::string hostname = "127.0.0.1";- std::string public_path = "/zip/llama.cpp/server/public";- std::string chat_template = "";- std::string system_prompt = "";+static bool ends_with(const std::string &str, const std::string &suffix)+{+ return str.size() >= suffix.size() &&+ 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);+}- std::vector<std::string> api_keys;+static size_t find_partial_stop_string(const std::string &stop,+ const std::string &text)+{+ if (!text.empty() && !stop.empty())+ {+ const char text_last_char = text.back();+ for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--)+ {+ if (stop[char_index] == text_last_char)+ {+ const std::string current_partial = stop.substr(0, char_index + 1);+ if (ends_with(text, current_partial))+ {+ return text.size() - char_index - 1;+ }+ }+ }+ }+ return std::string::npos;+}-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- std::string ssl_key_file = "";- std::string ssl_cert_file = "";-#endif+// TODO: reuse llama_detokenize+template <class Iter>+static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end)+{+ std::string ret;+ for (; begin != end; ++begin)+ {+ ret += llama_token_to_piece(ctx, *begin);+ }+ return ret;+}- bool slots_endpoint = true;- bool metrics_endpoint = false;+// format incomplete utf-8 multibyte character for output+static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token)+{+ std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token);+ // if the size is 1 and first bit is 1, meaning it's a partial character+ // (size > 1 meaning it's already a known token)+ if (out.size() == 1 && (out[0] & 0x80) == 0x80)+ {+ std::stringstream ss;+ ss << std::hex << (out[0] & 0xff);+ std::string res(ss.str());+ out = "byte: \\x" + res;+ }+ return out;+}- bool nobrowser = false;-};+// convert a vector of completion_token_output to json+static json probs_vector_to_json(const llama_context *ctx, const std::vector<completion_token_output> &probs)+{+ json out = json::array();+ for (const auto &prob : probs)+ {+ json probs_for_token = json::array();+ for (const auto &p : prob.probs)+ {+ std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);+ probs_for_token.push_back(json+ {+ {"tok_str", tok_str},+ {"prob", p.prob},+ });+ }+ std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);+ out.push_back(json{+ {"content", tok_str},+ {"probs", probs_for_token},+ });+ }+ return out;+}-struct server_slot {+struct llama_client_slot+{int id;- int id_task = -1;- int id_multi = -1;+ int task_id = -1;struct slot_params params;- slot_state state = SLOT_STATE_IDLE;- slot_command command = SLOT_COMMAND_NONE;+ slot_state state = IDLE;+ slot_command command = NONE;// used to determine the slot that has been used the longestint64_t t_last_used = -1;@@ -146,38 +157,33 @@ struct server_slot {int32_t n_decoded = 0;int32_t n_remaining = -1;int32_t i_batch = -1;- int32_t n_predict = -1; // TODO: disambiguate from params.n_predict+ int32_t n_predict = -1;- int32_t n_prompt_tokens = 0;- int32_t n_prompt_tokens_processed = 0;+ int32_t num_prompt_tokens = 0;+ int32_t num_prompt_tokens_processed = 0;json prompt;-- // when a task is submitted, we first tokenize the prompt and store it here- std::vector<llama_token> prompt_tokens;-std::string generated_text;+ llama_token sampled;std::vector<llama_token> cache_tokens;std::vector<completion_token_output> generated_token_probs;- bool infill = false;- bool embedding = false;+ bool infill = false;+ bool embedding = false;bool has_next_token = true;- bool truncated = false;- bool stopped_eos = false;- bool stopped_word = false;- bool stopped_limit = false;+ bool truncated = false;+ bool stopped_eos = false;+ bool stopped_word = false;+ bool stopped_limit = false;bool oaicompat = false;-std::string oaicompat_model;+std::string stopping_word;// sampling- llama_token sampled;struct llama_sampling_params sparams;- llama_sampling_context * ctx_sampling = nullptr;- json json_schema;+ llama_sampling_context *ctx_sampling = nullptr;int32_t ga_i = 0; // group-attention stateint32_t ga_n = 1; // group-attention factor@@ -185,44 +191,65 @@ struct server_slot {int32_t n_past_se = 0; // self-extend+ // multimodal+ std::vector<slot_image> images;+// stats- size_t n_sent_text = 0; // number of sent text character- size_t n_sent_token_probs = 0;+ size_t sent_count = 0;+ size_t sent_token_probs_index = 0;int64_t t_start_process_prompt;- int64_t t_start_generation;+ int64_t t_start_genereration;double t_prompt_processing; // msdouble t_token_generation; // ms+ // multitasks+ int multitask_id = -1;+void reset() {- n_prompt_tokens = 0;- generated_text = "";- truncated = false;- stopped_eos = false;- stopped_word = false;- stopped_limit = false;- stopping_word = "";- n_past = 0;- n_sent_text = 0;- n_sent_token_probs = 0;- infill = false;- ga_i = 0;- n_past_se = 0;+ num_prompt_tokens = 0;+ generated_text = "";+ truncated = false;+ stopped_eos = false;+ stopped_word = false;+ stopped_limit = false;+ stopping_word = "";+ n_past = 0;+ sent_count = 0;+ sent_token_probs_index = 0;+ infill = false;+ ga_i = 0;+ n_past_se = 0;generated_token_probs.clear();++ for (slot_image & img : images)+ {+ free(img.image_embedding);+ if (img.img_data) {+ clip_image_u8_free(img.img_data);+ }+ img.prefix_prompt = "";+ }++ images.clear();}bool has_budget(gpt_params &global_params) {- if (params.n_predict == -1 && global_params.n_predict == -1) {+ if (params.n_predict == -1 && global_params.n_predict == -1)+ {return true; // limitless}n_remaining = -1;- if (params.n_predict != -1) {+ if (params.n_predict != -1)+ {n_remaining = params.n_predict - n_decoded;- } else if (global_params.n_predict != -1) {+ }+ else if (global_params.n_predict != -1)+ {n_remaining = global_params.n_predict - n_decoded;}@@ -230,33 +257,37 @@ struct server_slot {}bool available() const {- return state == SLOT_STATE_IDLE && command == SLOT_COMMAND_NONE;+ return state == IDLE && command == NONE;}bool is_processing() const {- return (state == SLOT_STATE_IDLE && command == SLOT_COMMAND_LOAD_PROMPT) || state == SLOT_STATE_PROCESSING;+ return (state == IDLE && command == LOAD_PROMPT) || state == PROCESSING;}- void add_token_string(const completion_token_output & token) {- if (command == SLOT_COMMAND_RELEASE) {+ void add_token_string(const completion_token_output &token) {+ if (command == RELEASE)+ {return;}+ cache_tokens.push_back(token.tok);generated_token_probs.push_back(token);}void release() {- if (state == SLOT_STATE_PROCESSING) {- t_token_generation = (ggml_time_us() - t_start_generation) / 1e3;- command = SLOT_COMMAND_RELEASE;+ if (state == PROCESSING)+ {+ t_token_generation = (ggml_time_us() - t_start_genereration) / 1e3;+ command = RELEASE;}}- json get_formated_timings() const {- return json {- {"prompt_n", n_prompt_tokens_processed},+ json get_formated_timings() {+ return json+ {+ {"prompt_n", num_prompt_tokens_processed},{"prompt_ms", t_prompt_processing},- {"prompt_per_token_ms", t_prompt_processing / n_prompt_tokens_processed},- {"prompt_per_second", 1e3 / t_prompt_processing * n_prompt_tokens_processed},+ {"prompt_per_token_ms", t_prompt_processing / num_prompt_tokens_processed},+ {"prompt_per_second", 1e3 / t_prompt_processing * num_prompt_tokens_processed},{"predicted_n", n_decoded},{"predicted_ms", t_token_generation},@@ -265,74 +296,40 @@ struct server_slot {};}- size_t find_stopping_strings(const std::string & text, const size_t last_token_size, const stop_type type) {- size_t stop_pos = std::string::npos;-- for (const std::string & word : params.antiprompt) {- size_t pos;-- if (type == STOP_TYPE_FULL) {- const size_t tmp = word.size() + last_token_size;- const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;-- pos = text.find(word, from_pos);- } else {- pos = find_partial_stop_string(word, text);- }-- if (pos != std::string::npos && (stop_pos == std::string::npos || pos < stop_pos)) {- if (type == STOP_TYPE_FULL) {- stopped_word = true;- stopping_word = word;- has_next_token = false;- }- stop_pos = pos;- }- }-- return stop_pos;- }-void print_timings() const {- char buffer[512];-- double t_token = t_prompt_processing / n_prompt_tokens_processed;- double n_tokens_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;-- snprintf(buffer, 512, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)",- t_prompt_processing, n_prompt_tokens_processed,+ char buffer[512];+ double t_token = t_prompt_processing / num_prompt_tokens_processed;+ double n_tokens_second = 1e3 / t_prompt_processing * num_prompt_tokens_processed;+ sprintf(buffer, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)",+ t_prompt_processing, num_prompt_tokens_processed,t_token, n_tokens_second);-LOG_INFO(buffer, {- {"id_slot", id},- {"id_task", id_task},- {"t_prompt_processing", t_prompt_processing},- {"n_prompt_tokens_processed", n_prompt_tokens_processed},- {"t_token", t_token},- {"n_tokens_second", n_tokens_second},+ {"slot_id", id},+ {"task_id", task_id},+ {"t_prompt_processing", t_prompt_processing},+ {"num_prompt_tokens_processed", num_prompt_tokens_processed},+ {"t_token", t_token},+ {"n_tokens_second", n_tokens_second},});t_token = t_token_generation / n_decoded;n_tokens_second = 1e3 / t_token_generation * n_decoded;-- snprintf(buffer, 512, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)",+ sprintf(buffer, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)",t_token_generation, n_decoded,t_token, n_tokens_second);-LOG_INFO(buffer, {- {"id_slot", id},- {"id_task", id_task},+ {"slot_id", id},+ {"task_id", task_id},{"t_token_generation", t_token_generation},{"n_decoded", n_decoded},{"t_token", t_token},{"n_tokens_second", n_tokens_second},});- snprintf(buffer, 512, " total time = %10.2f ms", t_prompt_processing + t_token_generation);-+ sprintf(buffer, " total time = %10.2f ms", t_prompt_processing + t_token_generation);LOG_INFO(buffer, {- {"id_slot", id},- {"id_task", id_task},+ {"slot_id", id},+ {"task_id", task_id},{"t_prompt_processing", t_prompt_processing},{"t_token_generation", t_token_generation},{"t_total", t_prompt_processing + t_token_generation},@@ -340,36 +337,29 @@ struct server_slot {}};-struct server_metrics {- int64_t t_start = 0;-+struct llama_metrics {uint64_t n_prompt_tokens_processed_total = 0;- uint64_t t_prompt_processing_total = 0;uint64_t n_tokens_predicted_total = 0;- uint64_t t_tokens_generation_total = 0;uint64_t n_prompt_tokens_processed = 0;uint64_t t_prompt_processing = 0;- uint64_t n_tokens_predicted = 0;- uint64_t t_tokens_generation = 0;+ uint64_t n_tokens_predicted = 0;+ uint64_t t_tokens_generation = 0;- void init() {- t_start = ggml_time_us();- }- void on_prompt_eval(const server_slot & slot) {- n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed;- n_prompt_tokens_processed += slot.n_prompt_tokens_processed;- t_prompt_processing += slot.t_prompt_processing;- t_prompt_processing_total += slot.t_prompt_processing;+ void on_prompt_eval(const llama_client_slot &slot) {+ n_prompt_tokens_processed_total += slot.num_prompt_tokens_processed;++ n_prompt_tokens_processed += slot.num_prompt_tokens_processed;+ t_prompt_processing += slot.t_prompt_processing;}- void on_prediction(const server_slot & slot) {- n_tokens_predicted_total += slot.n_decoded;- n_tokens_predicted += slot.n_decoded;- t_tokens_generation += slot.t_token_generation;- t_tokens_generation_total += slot.t_token_generation;+ void on_prediction(const llama_client_slot &slot) {+ n_tokens_predicted_total += slot.n_decoded;++ n_tokens_predicted += slot.n_decoded;+ t_tokens_generation += slot.t_token_generation;}void reset_bucket() {@@ -380,261 +370,23 @@ struct server_metrics {}};-struct server_queue {- int id = 0;- bool running;-- // queues- std::vector<server_task> queue_tasks;- std::vector<server_task> queue_tasks_deferred;-- std::vector<server_task_multi> queue_multitasks;-- std::mutex mutex_tasks;- std::condition_variable condition_tasks;-- // callback functions- std::function<void(server_task &)> callback_new_task;- std::function<void(server_task_multi &)> callback_finish_multitask;- std::function<void(void)> callback_update_slots;-- // Add a new task to the end of the queue- int post(server_task task) {- std::unique_lock<std::mutex> lock(mutex_tasks);- if (task.id == -1) {- task.id = id++;- LOG_VERBOSE("new task id", {{"new_id", task.id}});- }- queue_tasks.push_back(std::move(task));- condition_tasks.notify_one();- return task.id;- }-- // Add a new task, but defer until one slot is available- void defer(server_task task) {- std::unique_lock<std::mutex> lock(mutex_tasks);- queue_tasks_deferred.push_back(std::move(task));- }-- // Get the next id for creating anew task- int get_new_id() {- std::unique_lock<std::mutex> lock(mutex_tasks);- int new_id = id++;- LOG_VERBOSE("new task id", {{"new_id", new_id}});- return new_id;- }-- // Register function to process a new task- void on_new_task(std::function<void(server_task &)> callback) {- callback_new_task = std::move(callback);- }-- // Register function to process a multitask when it is finished- void on_finish_multitask(std::function<void(server_task_multi&)> callback) {- callback_finish_multitask = std::move(callback);- }-- // Register the function to be called when all slots data is ready to be processed- void on_update_slots(std::function<void(void)> callback) {- callback_update_slots = std::move(callback);- }-- // Call when the state of one slot is changed- void notify_slot_changed() {- // move deferred tasks back to main loop- std::unique_lock<std::mutex> lock(mutex_tasks);- for (auto & task : queue_tasks_deferred) {- queue_tasks.push_back(std::move(task));- }- queue_tasks_deferred.clear();- }-- // end the start_loop routine- void terminate() {- std::unique_lock<std::mutex> lock(mutex_tasks);- running = false;- condition_tasks.notify_all();- }-- /**- * Main loop consists of these steps:- * - Wait until a new task arrives- * - Process the task (i.e. maybe copy data into slot)- * - Check if multitask is finished- * - Update all slots- */- void start_loop() {- running = true;-- while (true) {- LOG_VERBOSE("new task may arrive", {});-- while (true) {- std::unique_lock<std::mutex> lock(mutex_tasks);- if (queue_tasks.empty()) {- lock.unlock();- break;- }- server_task task = queue_tasks.front();- queue_tasks.erase(queue_tasks.begin());- lock.unlock();- LOG_VERBOSE("callback_new_task", {{"id_task", task.id}});- callback_new_task(task);- }-- LOG_VERBOSE("update_multitasks", {});-- // check if we have any finished multitasks- auto queue_iterator = queue_multitasks.begin();- while (queue_iterator != queue_multitasks.end()) {- if (queue_iterator->subtasks_remaining.empty()) {- // all subtasks done == multitask is done- server_task_multi current_multitask = *queue_iterator;- callback_finish_multitask(current_multitask);- // remove this multitask- queue_iterator = queue_multitasks.erase(queue_iterator);- } else {- ++queue_iterator;- }- }-- // all tasks in the current loop is processed, slots data is now ready- LOG_VERBOSE("callback_update_slots", {});-- callback_update_slots();-- LOG_VERBOSE("wait for new task", {});- {- std::unique_lock<std::mutex> lock(mutex_tasks);- if (queue_tasks.empty()) {- if (!running) {- LOG_VERBOSE("ending start_loop", {});- return;- }- condition_tasks.wait(lock, [&]{- return (!queue_tasks.empty() || !running);- });- }- }- }- }-- //- // functions to manage multitasks- //-- // add a multitask by specifying the id of all subtask (subtask is a server_task)- void add_multitask(int id_multi, std::vector<int> & sub_ids) {- std::lock_guard<std::mutex> lock(mutex_tasks);- server_task_multi multi;- multi.id = id_multi;- std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));- queue_multitasks.push_back(multi);- }-- // updatethe remaining subtasks, while appending results to multitask- void update_multitask(int id_multi, int id_sub, server_task_result & result) {- std::lock_guard<std::mutex> lock(mutex_tasks);- for (auto & multitask : queue_multitasks) {- if (multitask.id == id_multi) {- multitask.subtasks_remaining.erase(id_sub);- multitask.results.push_back(result);- }- }- }-};--struct server_response {- typedef std::function<void(int, int, server_task_result &)> callback_multitask_t;- callback_multitask_t callback_update_multitask;-- // for keeping track of all tasks waiting for the result- std::set<int> waiting_task_ids;-- // the main result queue- std::vector<server_task_result> queue_results;-- std::mutex mutex_results;- std::condition_variable condition_results;-- // add the id_task to the list of tasks waiting for response- void add_waiting_task_id(int id_task) {- LOG_VERBOSE("waiting for task id", {{"id_task", id_task}});-- std::unique_lock<std::mutex> lock(mutex_results);- waiting_task_ids.insert(id_task);- }-- // when the request is finished, we can remove task associated with it- void remove_waiting_task_id(int id_task) {- LOG_VERBOSE("remove waiting for task id", {{"id_task", id_task}});-- std::unique_lock<std::mutex> lock(mutex_results);- waiting_task_ids.erase(id_task);- }-- // This function blocks the thread until there is a response for this id_task- server_task_result recv(int id_task) {- while (true) {- std::unique_lock<std::mutex> lock(mutex_results);- condition_results.wait(lock, [&]{- return !queue_results.empty();- });-- for (int i = 0; i < (int) queue_results.size(); i++) {- if (queue_results[i].id == id_task) {- assert(queue_results[i].id_multi == -1);- server_task_result res = queue_results[i];- queue_results.erase(queue_results.begin() + i);- return res;- }- }- }-- // should never reach here- }-- // Register the function to update multitask- void on_multitask_update(callback_multitask_t callback) {- callback_update_multitask = std::move(callback);- }-- // Send a new result to a waiting id_task- void send(server_task_result result) {- LOG_VERBOSE("send new result", {{"id_task", result.id}});-- std::unique_lock<std::mutex> lock(mutex_results);- for (const auto & id_task : waiting_task_ids) {- // LOG_TEE("waiting task id %i \n", id_task);- // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result- if (result.id_multi == id_task) {- LOG_VERBOSE("callback_update_multitask", {{"id_task", id_task}});- callback_update_multitask(id_task, result.id, result);- continue;- }-- if (result.id == id_task) {- LOG_VERBOSE("queue_results.push_back", {{"id_task", id_task}});- queue_results.push_back(result);- condition_results.notify_all();- return;- }- }- }-};+struct llama_server_context+{+ llama_model *model = nullptr;+ llama_context *ctx = nullptr;-struct server_context {- llama_model * model = nullptr;- llama_context * ctx = nullptr;+ clip_ctx *clp_ctx = nullptr;gpt_params params;llama_batch batch;- bool clean_kv_cache = true;- bool add_bos_token = true;+ bool multimodal = false;+ bool clean_kv_cache = true;+ bool all_slots_are_idle = false;+ bool add_bos_token = true;- int32_t n_ctx; // total context for all clients / slots+ int32_t n_ctx; // total context for all clients / slots// system promptbool system_need_update = false;@@ -646,39 +398,63 @@ struct server_context {std::string name_assistant;// slots / clients- std::vector<server_slot> slots;+ std::vector<llama_client_slot> slots;json default_generation_settings_for_props;- server_queue queue_tasks;- server_response queue_results;+ llama_server_queue queue_tasks;+ llama_server_response queue_results;- server_metrics metrics;+ llama_metrics metrics;- ~server_context() {- if (ctx) {+ ~llama_server_context()+ {+ if (ctx)+ {llama_free(ctx);ctx = nullptr;}-- if (model) {+ if (model)+ {llama_free_model(model);model = nullptr;}}- bool load_model(const gpt_params & params_) {+ bool load_model(const gpt_params ¶ms_)+ {params = params_;+ if (!params.mmproj.empty()) {+ multimodal = true;+ LOG_INFO("Multi Modal Mode Enabled", {});+ clp_ctx = clip_model_load(params.mmproj.c_str(), /*verbosity=*/ 1);+ if(clp_ctx == nullptr) {+ LOG_ERROR("unable to load clip model", {{"model", params.mmproj}});+ return false;+ }- // dedicate one sequence to the system prompt- params.n_parallel += 1;+ if (params.n_ctx < 2048) { // request larger context for the image embedding+ params.n_ctx = 2048;+ }+ }std::tie(model, ctx) = llama_init_from_gpt_params(params);- params.n_parallel -= 1; // but be sneaky about it- if (model == nullptr) {+ if (model == nullptr)+ {LOG_ERROR("unable to load model", {{"model", params.model}});return false;}+ if (multimodal) {+ const int n_embd_clip = clip_n_mmproj_embd(clp_ctx);+ const int n_embd_llm = llama_n_embd(model);+ if (n_embd_clip != n_embd_llm) {+ LOG_TEE("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_embd_clip, n_embd_llm);+ llama_free(ctx);+ llama_free_model(model);+ return false;+ }+ }+n_ctx = llama_n_ctx(ctx);add_bos_token = llama_should_add_bos_token(model);@@ -686,28 +462,33 @@ struct server_context {return true;}- bool validate_model_chat_template() const {+ void validate_model_chat_template(server_params & sparams) {llama_chat_message chat[] = {{"user", "test"}};-- const int res = llama_chat_apply_template(model, nullptr, chat, 1, true, nullptr, 0);-- return res > 0;+ std::vector<char> buf(1);+ int res = llama_chat_apply_template(model, nullptr, chat, 1, true, buf.data(), buf.size());+ if (res < 0) {+ LOG_ERROR("The chat template comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {});+ sparams.chat_template = "<|im_start|>"; // llama_chat_apply_template only checks if <|im_start|> exist in the template+ }}- void init() {+ void initialize() {+ // create slots+ all_slots_are_idle = true;+const int32_t n_ctx_slot = n_ctx / params.n_parallel;LOG_INFO("initializing slots", {{"n_slots", params.n_parallel}});-- for (int i = 0; i < params.n_parallel; i++) {- server_slot slot;+ for (int i = 0; i < params.n_parallel; i++)+ {+ llama_client_slot slot;slot.id = i;slot.n_ctx = n_ctx_slot;slot.n_predict = params.n_predict;LOG_INFO("new slot", {- {"id_slot", slot.id},+ {"slot_id", slot.id},{"n_ctx_slot", slot.n_ctx}});@@ -715,15 +496,15 @@ struct server_context {const int ga_w = params.grp_attn_w;if (ga_n != 1) {- GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT- GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT+ GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT+ GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT//GGML_ASSERT(n_ctx_train % ga_w == 0 && "n_ctx_train must be a multiple of ga_w"); // NOLINT//GGML_ASSERT(n_ctx >= n_ctx_train * ga_n && "n_ctx must be at least n_ctx_train * ga_n"); // NOLINTLOG_INFO("slot self-extend", {- {"id_slot", slot.id},- {"ga_n", ga_n},- {"ga_w", ga_w}+ {"slot_id", slot.id},+ {"ga_n", ga_n},+ {"ga_w", ga_w}});}@@ -739,18 +520,11 @@ struct server_context {default_generation_settings_for_props = get_formated_generation(slots.front());default_generation_settings_for_props["seed"] = -1;- // the update_slots() logic will always submit a maximum of n_batch tokens- // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used)- {- const int32_t n_batch = llama_n_batch(ctx);-- batch = llama_batch_init(n_batch, 0, params.n_parallel);- }-- metrics.init();+ batch = llama_batch_init(n_ctx, 0, params.n_parallel);}- std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const {+ std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const+ {// TODO: currently, we tokenize using special tokens by default// this is not always correct (see https://github.com/ggerganov/llama.cpp/pull/4160#issuecomment-1824826216)// but it's better compared to completely ignoring ChatML and other chat templates@@ -760,30 +534,38 @@ struct server_context {// or the first element of the json_prompt array is a string.std::vector<llama_token> prompt_tokens;- if (json_prompt.is_array()) {+ if (json_prompt.is_array())+ {bool first = true;- for (const auto & p : json_prompt) {- if (p.is_string()) {+ for (const auto& p : json_prompt)+ {+ if (p.is_string())+ {auto s = p.template get<std::string>();-std::vector<llama_token> p;- if (first) {+ if (first)+ {p = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);first = false;- } else {+ }+ else+ {p = ::llama_tokenize(ctx, s, false, TMP_FORCE_SPECIAL);}-prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());- } else {- if (first) {+ }+ else+ {+ if (first)+ {first = false;}-prompt_tokens.push_back(p.template get<llama_token>());}}- } else {+ }+ else+ {auto s = json_prompt.template get<std::string>();prompt_tokens = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);}@@ -791,18 +573,19 @@ struct server_context {return prompt_tokens;}- server_slot * get_slot(int id) {+ llama_client_slot* get_slot(int id) {int64_t t_last = ggml_time_us();+ llama_client_slot *last_used = nullptr;- server_slot * last_used = nullptr;-- for (server_slot & slot : slots) {- if (slot.id == id && slot.available()) {+ for (llama_client_slot & slot : slots)+ {+ if (slot.id == id && slot.available())+ {return &slot;}- // among all available slots, find the one that has been least recently used- if (slot.available() && slot.t_last_used < t_last) {+ if (slot.available() && slot.t_last_used < t_last)+ {last_used = &slot;t_last = slot.t_last_used;}@@ -811,235 +594,295 @@ struct server_context {return last_used;}- bool launch_slot_with_task(server_slot & slot, const server_task & task) {+ bool launch_slot_with_data(llama_client_slot* &slot, json data) {slot_params default_params;llama_sampling_params default_sparams;- auto & data = task.data;if (data.count("__oaicompat") != 0) {- slot.oaicompat = true;- slot.oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));- } else {- slot.oaicompat = false;- slot.oaicompat_model = "";- }-- slot.params.stream = json_value(data, "stream", false);- slot.params.cache_prompt = json_value(data, "cache_prompt", false);- slot.params.n_predict = json_value(data, "n_predict", default_params.n_predict);- slot.sparams.top_k = json_value(data, "top_k", default_sparams.top_k);- slot.sparams.top_p = json_value(data, "top_p", default_sparams.top_p);- slot.sparams.min_p = json_value(data, "min_p", default_sparams.min_p);- slot.sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);- slot.sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);- slot.sparams.temp = json_value(data, "temperature", default_sparams.temp);- slot.sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);- slot.sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);- slot.sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);- slot.sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);- slot.sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);- slot.sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);- slot.sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);- slot.sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);- slot.sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);- slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);- slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep);- slot.params.seed = json_value(data, "seed", default_params.seed);- if (data.contains("json_schema") && !data.contains("grammar")) {- try {- auto schema = json_value(data, "json_schema", json::object());- slot.sparams.grammar = json_schema_to_grammar(schema);- } catch (const std::exception & e) {- send_error(task, std::string("\"json_schema\": ") + e.what(), ERROR_TYPE_INVALID_REQUEST);- return false;- }+ slot->oaicompat = true;+ slot->oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));} else {- slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar);- }- slot.sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);- slot.sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);-- if (slot.params.cache_prompt && slot.ga_n != 1) {- LOG_WARNING("cache_prompt is not supported with group-attention", {});- slot.params.cache_prompt = false;- }-- if (slot.n_predict > 0 && slot.params.n_predict > slot.n_predict) {+ slot->oaicompat = false;+ slot->oaicompat_model = "";+ }++ slot->params.stream = json_value(data, "stream", false);+ slot->params.cache_prompt = json_value(data, "cache_prompt", false);+ slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict);+ slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);+ slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);+ slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);+ slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);+ slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);+ slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);+ slot->sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);+ slot->sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);+ slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);+ slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);+ slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);+ slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);+ slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);+ slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);+ slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);+ slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);+ slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep);+ slot->params.seed = json_value(data, "seed", default_params.seed);+ slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar);+ slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);+ slot->sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);++ if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) {// Might be better to reject the request with a 400 ?LOG_WARNING("Max tokens to predict exceeds server configuration", {- {"params.n_predict", slot.params.n_predict},- {"slot.n_predict", slot.n_predict},+ {"params.n_predict", slot->params.n_predict},+ {"slot.n_predict", slot->n_predict},});- slot.params.n_predict = slot.n_predict;+ slot->params.n_predict = slot->n_predict;}// infill- slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix);- slot.params.input_suffix = json_value(data, "input_suffix", default_params.input_suffix);-- // get prompt+ if (data.count("input_prefix") != 0){- const auto & prompt = data.find("prompt");- if (prompt == data.end()) {- send_error(task, "Either \"prompt\" or \"messages\" must be provided", ERROR_TYPE_INVALID_REQUEST);- return false;- } else {- slot.prompt = *prompt;- }- if (slot.prompt.is_array() && slot.prompt.size() == 0) {- send_error(task, "\"prompt\" cannot be an empty array", ERROR_TYPE_INVALID_REQUEST);- return false;- }+ slot->params.input_prefix = data["input_prefix"];}-- // penalize user-provided tokens+ else{- slot.sparams.penalty_prompt_tokens.clear();- slot.sparams.use_penalty_prompt_tokens = false;-- const auto & penalty_prompt = data.find("penalty_prompt");+ slot->params.input_prefix = "";+ }- if (penalty_prompt != data.end()) {- if (penalty_prompt->is_string()) {- const auto penalty_prompt_string = penalty_prompt->get<std::string>();- slot.sparams.penalty_prompt_tokens = llama_tokenize(model, penalty_prompt_string, false);+ if (data.count("input_suffix") != 0)+ {+ slot->params.input_suffix = data["input_suffix"];+ }+ else+ {+ slot->params.input_suffix = "";+ }- if (slot.params.n_predict > 0) {- slot.sparams.penalty_prompt_tokens.reserve(slot.sparams.penalty_prompt_tokens.size() + slot.params.n_predict);- }- slot.sparams.use_penalty_prompt_tokens = true;+ if (data.count("prompt") != 0)+ {+ slot->prompt = data["prompt"];+ }+ else+ {+ slot->prompt = "";+ }- LOG_VERBOSE("penalty_prompt_tokens", {- {"id_slot", slot.id},- {"tokens", slot.sparams.penalty_prompt_tokens},- });+ slot->sparams.penalty_prompt_tokens.clear();+ slot->sparams.use_penalty_prompt_tokens = false;+ const auto &penalty_prompt = data.find("penalty_prompt");+ if (penalty_prompt != data.end())+ {+ if (penalty_prompt->is_string())+ {+ const auto penalty_prompt_string = penalty_prompt->get<std::string>();+ auto penalty_tokens = llama_tokenize(model, penalty_prompt_string, false);+ slot->sparams.penalty_prompt_tokens.swap(penalty_tokens);+ if (slot->params.n_predict > 0)+ {+ slot->sparams.penalty_prompt_tokens.reserve(slot->sparams.penalty_prompt_tokens.size() + slot->params.n_predict);}- else if (penalty_prompt->is_array()) {- const auto n_tokens = penalty_prompt->size();- slot.sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot.params.n_predict));-- const int n_vocab = llama_n_vocab(model);- for (const auto & penalty_token : *penalty_prompt) {- if (penalty_token.is_number_integer()) {- const auto tok = penalty_token.get<llama_token>();- if (tok >= 0 && tok < n_vocab) {- slot.sparams.penalty_prompt_tokens.push_back(tok);- }+ slot->sparams.use_penalty_prompt_tokens = true;+ }+ else if (penalty_prompt->is_array())+ {+ const auto n_tokens = penalty_prompt->size();+ slot->sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot->params.n_predict));+ const int n_vocab = llama_n_vocab(model);+ for (const auto &penalty_token : *penalty_prompt)+ {+ if (penalty_token.is_number_integer())+ {+ const auto tok = penalty_token.get<llama_token>();+ if (tok >= 0 && tok < n_vocab)+ {+ slot->sparams.penalty_prompt_tokens.push_back(tok);}}- slot.sparams.use_penalty_prompt_tokens = true;-- LOG_VERBOSE("penalty_prompt_tokens", {- {"id_slot", slot.id},- {"tokens", slot.sparams.penalty_prompt_tokens},- });}+ slot->sparams.use_penalty_prompt_tokens = true;}}+ slot->sparams.logit_bias.clear();++ if (json_value(data, "ignore_eos", false)){- slot.sparams.logit_bias.clear();+ slot->sparams.logit_bias[llama_token_eos(model)] = -INFINITY;+ }- if (json_value(data, "ignore_eos", false)) {- slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;- }+ const auto &logit_bias = data.find("logit_bias");+ if (logit_bias != data.end() && logit_bias->is_array())+ {+ const int n_vocab = llama_n_vocab(model);+ for (const auto &el : *logit_bias)+ {+ if (el.is_array() && el.size() == 2)+ {+ float bias;+ if (el[1].is_number())+ {+ bias = el[1].get<float>();+ }+ else if (el[1].is_boolean() && !el[1].get<bool>())+ {+ bias = -INFINITY;+ }+ else+ {+ continue;+ }- const auto & logit_bias = data.find("logit_bias");- if (logit_bias != data.end() && logit_bias->is_array()) {- const int n_vocab = llama_n_vocab(model);- for (const auto & el : *logit_bias) {- // TODO: we may want to throw errors here, in case "el" is incorrect- if (el.is_array() && el.size() == 2) {- float bias;- if (el[1].is_number()) {- bias = el[1].get<float>();- } else if (el[1].is_boolean() && !el[1].get<bool>()) {- bias = -INFINITY;- } else {- continue;+ if (el[0].is_number_integer())+ {+ llama_token tok = el[0].get<llama_token>();+ if (tok >= 0 && tok < n_vocab)+ {+ slot->sparams.logit_bias[tok] = bias;}-- if (el[0].is_number_integer()) {- llama_token tok = el[0].get<llama_token>();- if (tok >= 0 && tok < n_vocab) {- slot.sparams.logit_bias[tok] = bias;- }- } else if (el[0].is_string()) {- auto toks = llama_tokenize(model, el[0].get<std::string>(), false);- for (auto tok : toks) {- slot.sparams.logit_bias[tok] = bias;- }+ }+ else if (el[0].is_string())+ {+ auto toks = llama_tokenize(model, el[0].get<std::string>(), false);+ for (auto tok : toks)+ {+ slot->sparams.logit_bias[tok] = bias;}}}}}+ slot->params.antiprompt.clear();++ const auto &stop = data.find("stop");+ if (stop != data.end() && stop->is_array()){- slot.params.antiprompt.clear();+ for (const auto &word : *stop)+ {+ if (!word.empty())+ {+ slot->params.antiprompt.push_back(word);+ }+ }+ }- const auto & stop = data.find("stop");- if (stop != data.end() && stop->is_array()) {- for (const auto & word : *stop) {- if (!word.empty()) {- slot.params.antiprompt.push_back(word);- }+ const auto &samplers_sequence = data.find("samplers");+ if (samplers_sequence != data.end() && samplers_sequence->is_array())+ {+ std::vector<std::string> sampler_names;+ for (const auto &sampler_name : *samplers_sequence)+ {+ if (sampler_name.is_string())+ {+ sampler_names.emplace_back(sampler_name);}}+ slot->sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);+ }+ else+ {+ slot->sparams.samplers_sequence = default_sparams.samplers_sequence;}+ if (multimodal){- const auto & samplers_sequence = data.find("samplers");- if (samplers_sequence != data.end() && samplers_sequence->is_array()) {- std::vector<std::string> sampler_names;- for (const auto & sampler_name : *samplers_sequence) {- if (sampler_name.is_string()) {- sampler_names.emplace_back(sampler_name);+ const auto &images_data = data.find("image_data");+ if (images_data != data.end() && images_data->is_array())+ {+ for (const auto &img : *images_data)+ {+ const std::vector<uint8_t> image_buffer = base64_decode(img["data"].get<std::string>());++ slot_image img_sl;+ img_sl.id = img.count("id") != 0 ? img["id"].get<int>() : slot->images.size();+ img_sl.img_data = clip_image_u8_init();+ if (!clip_image_load_from_bytes(image_buffer.data(), image_buffer.size(), img_sl.img_data))+ {+ LOG_ERROR("failed to load image", {+ {"slot_id", slot->id},+ {"img_sl_id", img_sl.id}+ });+ return false;}+ LOG_VERBOSE("image loaded", {+ {"slot_id", slot->id},+ {"img_sl_id", img_sl.id}+ });+ img_sl.request_encode_image = true;+ slot->images.push_back(img_sl);+ }+ // process prompt+ // example: system prompt [img-102] user [img-103] describe [img-134] -> [{id: 102, prefix: 'system prompt '}, {id: 103, prefix: ' user '}, {id: 134, prefix: ' describe '}]}+ if (slot->images.size() > 0 && !slot->prompt.is_array())+ {+ std::string prompt = slot->prompt.get<std::string>();+ size_t pos = 0, begin_prefix = 0;+ std::string pattern = "[img-";+ while ((pos = prompt.find(pattern, pos)) != std::string::npos) {+ size_t end_prefix = pos;+ pos += pattern.length();+ size_t end_pos = prompt.find(']', pos);+ if (end_pos != std::string::npos)+ {+ std::string image_id = prompt.substr(pos, end_pos - pos);+ try+ {+ int img_id = std::stoi(image_id);+ bool found = false;+ for (slot_image &img : slot->images)+ {+ if (img.id == img_id) {+ found = true;+ img.prefix_prompt = prompt.substr(begin_prefix, end_prefix - begin_prefix);+ begin_prefix = end_pos + 1;+ break;+ }+ }+ if (!found) {+ LOG_TEE("ERROR: Image with id: %i, not found.\n", img_id);+ slot->images.clear();+ return false;+ }+ } catch (const std::invalid_argument& e) {+ LOG_TEE("Invalid image number id in prompt\n");+ slot->images.clear();+ return false;+ }+ }+ }+ slot->prompt = "";+ slot->params.input_suffix = prompt.substr(begin_prefix);+ slot->params.cache_prompt = false; // multimodal doesn't support cache prompt}- slot.sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);- } else {- slot.sparams.samplers_sequence = default_sparams.samplers_sequence;}}+ if (slot->ctx_sampling != nullptr){- if (slot.ctx_sampling != nullptr) {- llama_sampling_free(slot.ctx_sampling);- }- slot.ctx_sampling = llama_sampling_init(slot.sparams);- if (slot.ctx_sampling == nullptr) {- // for now, the only error that may happen here is invalid grammar- send_error(task, "Failed to parse grammar", ERROR_TYPE_INVALID_REQUEST);- return false;- }- llama_set_rng_seed(ctx, slot.params.seed);+ llama_sampling_free(slot->ctx_sampling);}+ slot->ctx_sampling = llama_sampling_init(slot->sparams);+ llama_set_rng_seed(ctx, slot->params.seed);+ slot->command = LOAD_PROMPT;- slot.command = SLOT_COMMAND_LOAD_PROMPT;- slot.prompt_tokens.clear();+ all_slots_are_idle = false;LOG_INFO("slot is processing task", {- {"id_slot", slot.id},- {"id_task", slot.id_task},+ {"slot_id", slot->id},+ {"task_id", slot->task_id},});return true;}void kv_cache_clear() {- LOG_VERBOSE("clearing KV cache", {});-// clear the entire KV cachellama_kv_cache_clear(ctx);clean_kv_cache = false;}- void system_prompt_update() {- LOG_VERBOSE("system prompt update", {- {"system_prompt", system_prompt},- });-+ void update_system_prompt() {kv_cache_clear();system_tokens.clear();@@ -1048,14 +891,14 @@ struct server_context {llama_batch_clear(batch);- for (int i = 0; i < (int)system_tokens.size(); ++i) {+ for (int i = 0; i < (int)system_tokens.size(); ++i)+ {llama_batch_add(batch, system_tokens[i], i, { 0 }, false);}- const int32_t n_batch = llama_n_batch(ctx);-- for (int32_t i = 0; i < batch.n_tokens; i += n_batch) {- const int32_t n_tokens = std::min(params.n_batch, batch.n_tokens - i);+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch)+ {+ const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i));llama_batch batch_view = {n_tokens,batch.token + i,@@ -1066,42 +909,78 @@ struct server_context {batch.logits + i,0, 0, 0, // unused};-- if (llama_decode(ctx, batch_view) != 0) {+ if (llama_decode(ctx, batch_view) != 0)+ {LOG_TEE("%s: llama_decode() failed\n", __func__);return;}}// assign the system KV cache to all parallel sequences- for (int32_t i = 1; i <= params.n_parallel; ++i) {- llama_kv_cache_seq_cp(ctx, 0, i, -1, -1);+ for (int32_t i = 1; i < params.n_parallel; ++i)+ {+ llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size());}}+ LOG_TEE("system prompt updated\n");system_need_update = false;}- void system_prompt_set(const json & sys_props) {+ void notify_system_prompt_changed() {+ // release all slots+ for (llama_client_slot &slot : slots)+ {+ slot.release();+ }++ system_need_update = true;+ }++ void process_system_prompt_data(const json &sys_props) {system_prompt = sys_props.value("prompt", "");name_user = sys_props.value("anti_prompt", "");name_assistant = sys_props.value("assistant_name", "");- LOG_VERBOSE("system prompt process", {- {"system_prompt", system_prompt},- {"name_user", name_user},- {"name_assistant", name_assistant},- });- // release all slots- for (server_slot & slot : slots) {- slot.release();+ notify_system_prompt_changed();+ }++ static size_t find_stopping_strings(const std::string &text, const size_t last_token_size,+ const stop_type type, llama_client_slot &slot)+ {+ size_t stop_pos = std::string::npos;++ for (const std::string &word : slot.params.antiprompt)+ {+ size_t pos;+ if (type == STOP_FULL)+ {+ const size_t tmp = word.size() + last_token_size;+ const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;+ pos = text.find(word, from_pos);+ }+ else+ {+ pos = find_partial_stop_string(word, text);+ }+ if (pos != std::string::npos &&+ (stop_pos == std::string::npos || pos < stop_pos))+ {+ if (type == STOP_FULL)+ {+ slot.stopped_word = true;+ slot.stopping_word = word;+ slot.has_next_token = false;+ }+ stop_pos = pos;+ }}- system_need_update = true;+ return stop_pos;}- bool process_token(completion_token_output & result, server_slot & slot) {+ bool process_token(completion_token_output &result, llama_client_slot &slot) {// remember which tokens were sampled - used for repetition penalties during samplingconst std::string token_str = llama_token_to_piece(ctx, result.tok);slot.sampled = result.tok;@@ -1110,26 +989,34 @@ struct server_context {slot.generated_text += token_str;slot.has_next_token = true;- if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1) {+ if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1)+ {// we can change penalty_prompt_tokens because it is always created from scratch each requestslot.ctx_sampling->params.penalty_prompt_tokens.push_back(result.tok);}// check if there is incomplete UTF-8 character at the endbool incomplete = false;- for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i) {+ for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i)+ {unsigned char c = slot.generated_text[slot.generated_text.size() - i];- if ((c & 0xC0) == 0x80) {+ if ((c & 0xC0) == 0x80)+ {// continuation byte: 10xxxxxxcontinue;}- if ((c & 0xE0) == 0xC0) {+ if ((c & 0xE0) == 0xC0)+ {// 2-byte character: 110xxxxx ...incomplete = i < 2;- } else if ((c & 0xF0) == 0xE0) {+ }+ else if ((c & 0xF0) == 0xE0)+ {// 3-byte character: 1110xxxx ...incomplete = i < 3;- } else if ((c & 0xF8) == 0xF0) {+ }+ else if ((c & 0xF8) == 0xF0)+ {// 4-byte character: 11110xxx ...incomplete = i < 4;}@@ -1137,196 +1024,210 @@ struct server_context {break;}- if (!incomplete) {- size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());-+ if (!incomplete)+ {+ size_t pos = std::min(slot.sent_count, slot.generated_text.size());const std::string str_test = slot.generated_text.substr(pos);bool is_stop_full = false;-- size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_FULL);- if (stop_pos != std::string::npos) {+ size_t stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_FULL, slot);+ if (stop_pos != std::string::npos)+ {is_stop_full = true;slot.generated_text.erase(slot.generated_text.begin() + pos + stop_pos,slot.generated_text.end());- pos = std::min(slot.n_sent_text, slot.generated_text.size());- } else {+ pos = std::min(slot.sent_count, slot.generated_text.size());+ }+ else+ {is_stop_full = false;- stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_PARTIAL);+ stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_PARTIAL, slot);}// check if there is any token to predict- if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0)) {+ if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0))+ {// no send the stop word in the responseresult.text_to_send = slot.generated_text.substr(pos, std::string::npos);- slot.n_sent_text += result.text_to_send.size();+ slot.sent_count += result.text_to_send.size();// add the token to slot queue and cache}-slot.add_token_string(result);- if (slot.params.stream) {+ if (slot.params.stream)+ {send_partial_response(slot, result);}}- if (incomplete) {+ if (incomplete)+ {slot.has_next_token = true;}// check the limits- if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params)) {- slot.stopped_limit = true;+ if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params))+ {+ slot.stopped_limit = true;slot.has_next_token = false;-- LOG_VERBOSE("stopped by limit", {- {"id_slot", slot.id},- {"id_task", slot.id_task},- {"n_decoded", slot.n_decoded},- {"n_predict", slot.params.n_predict},- });}- if (result.tok == llama_token_eos(model)) {- slot.stopped_eos = true;+ if (!slot.cache_tokens.empty() && result.tok == llama_token_eos(model))+ {+ slot.stopped_eos = true;slot.has_next_token = false;-LOG_VERBOSE("eos token found", {});}LOG_VERBOSE("next token", {- {"id_slot", slot.id},- {"id_task", slot.id_task},- {"token", result.tok},- {"token_text", tokens_to_output_formatted_string(ctx, result.tok)},- {"has_next_token", slot.has_next_token},- {"n_remain", slot.n_remaining},- {"n_decoded", slot.n_decoded},- {"stopped_eos", slot.stopped_eos},- {"stopped_word", slot.stopped_word},- {"stopped_limit", slot.stopped_limit},- {"stopping_word", slot.stopping_word},- });+ {"token", result.tok},+ {"token_text", tokens_to_output_formatted_string(ctx, result.tok)},+ {"has_next_token", slot.has_next_token},+ {"n_remain", slot.n_remaining},+ {"num_tokens_predicted", slot.n_decoded},+ {"stopped_eos", slot.stopped_eos},+ {"stopped_word", slot.stopped_word},+ {"stopped_limit", slot.stopped_limit},+ {"stopping_word", slot.stopping_word},+ });return slot.has_next_token; // continue}- json get_formated_generation(const server_slot & slot) const {- const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));- const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && eos_bias->second < 0.0f && std::isinf(eos_bias->second);+ bool process_images(llama_client_slot &slot) const+ {+ for (slot_image &img : slot.images)+ {+ if (!img.request_encode_image)+ {+ continue;+ }++ if (!llava_image_embed_make_with_clip_img(clp_ctx, params.n_threads, img.img_data, &img.image_embedding, &img.image_tokens)) {+ LOG_TEE("Error processing the given image");+ return false;+ }+++ img.request_encode_image = false;+ }++ return slot.images.size() > 0;+ }+ void send_error(task_server& task, const std::string &error)+ {+ LOG_TEE("task %i - error: %s\n", task.id, error.c_str());+ task_result res;+ res.id = task.id;+ res.multitask_id = task.multitask_id;+ res.stop = false;+ res.error = true;+ res.result_json = { { "content", error } };+ queue_results.send(res);+ }++ json get_formated_generation(llama_client_slot &slot)+ {+ const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));+ const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() &&+ eos_bias->second < 0.0f && std::isinf(eos_bias->second);std::vector<std::string> samplers_sequence;- samplers_sequence.reserve(slot.sparams.samplers_sequence.size());- for (const auto & sampler_type : slot.sparams.samplers_sequence) {+ for (const auto &sampler_type : slot.sparams.samplers_sequence)+ {samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type));}return json {- {"n_ctx", slot.n_ctx},- {"n_predict", slot.n_predict},- {"model", params.model_alias},- {"seed", slot.params.seed},- {"temperature", slot.sparams.temp},- {"dynatemp_range", slot.sparams.dynatemp_range},- {"dynatemp_exponent", slot.sparams.dynatemp_exponent},- {"top_k", slot.sparams.top_k},- {"top_p", slot.sparams.top_p},- {"min_p", slot.sparams.min_p},- {"tfs_z", slot.sparams.tfs_z},- {"typical_p", slot.sparams.typical_p},- {"repeat_last_n", slot.sparams.penalty_last_n},- {"repeat_penalty", slot.sparams.penalty_repeat},- {"presence_penalty", slot.sparams.penalty_present},- {"frequency_penalty", slot.sparams.penalty_freq},- {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},+ {"n_ctx", slot.n_ctx},+ {"n_predict", slot.n_predict},+ {"model", params.model_alias},+ {"seed", slot.params.seed},+ {"temperature", slot.sparams.temp},+ {"dynatemp_range", slot.sparams.dynatemp_range},+ {"dynatemp_exponent", slot.sparams.dynatemp_exponent},+ {"top_k", slot.sparams.top_k},+ {"top_p", slot.sparams.top_p},+ {"min_p", slot.sparams.min_p},+ {"tfs_z", slot.sparams.tfs_z},+ {"typical_p", slot.sparams.typical_p},+ {"repeat_last_n", slot.sparams.penalty_last_n},+ {"repeat_penalty", slot.sparams.penalty_repeat},+ {"presence_penalty", slot.sparams.penalty_present},+ {"frequency_penalty", slot.sparams.penalty_freq},+ {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},{"use_penalty_prompt_tokens", slot.sparams.use_penalty_prompt_tokens},- {"mirostat", slot.sparams.mirostat},- {"mirostat_tau", slot.sparams.mirostat_tau},- {"mirostat_eta", slot.sparams.mirostat_eta},- {"penalize_nl", slot.sparams.penalize_nl},- {"stop", slot.params.antiprompt},- {"n_predict", slot.params.n_predict}, // TODO: fix duplicate key n_predict- {"n_keep", params.n_keep},- {"ignore_eos", ignore_eos},- {"stream", slot.params.stream},- {"logit_bias", slot.sparams.logit_bias},- {"n_probs", slot.sparams.n_probs},- {"min_keep", slot.sparams.min_keep},- {"grammar", slot.sparams.grammar},- {"samplers", samplers_sequence}+ {"mirostat", slot.sparams.mirostat},+ {"mirostat_tau", slot.sparams.mirostat_tau},+ {"mirostat_eta", slot.sparams.mirostat_eta},+ {"penalize_nl", slot.sparams.penalize_nl},+ {"stop", slot.params.antiprompt},+ {"n_predict", slot.params.n_predict},+ {"n_keep", params.n_keep},+ {"ignore_eos", ignore_eos},+ {"stream", slot.params.stream},+ {"logit_bias", slot.sparams.logit_bias},+ {"n_probs", slot.sparams.n_probs},+ {"min_keep", slot.sparams.min_keep},+ {"grammar", slot.sparams.grammar},+ {"samplers", samplers_sequence}};}- void send_error(const server_task & task, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {- send_error(task.id, task.id_multi, error, type);- }-- void send_error(const server_slot & slot, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {- send_error(slot.id_task, slot.id_multi, error, type);- }-- void send_error(const int id_task, const int id_multi, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {- LOG_TEE("task %i - error: %s\n", id_task, error.c_str());-- server_task_result res;- res.id = id_task;- res.id_multi = id_multi;- res.stop = false;- res.error = true;- res.data = format_error_response(error, type);-- queue_results.send(res);- }+ void send_partial_response(llama_client_slot &slot, completion_token_output tkn)+ {+ task_result res;+ res.id = slot.task_id;+ res.multitask_id = slot.multitask_id;+ res.error = false;+ res.stop = false;- void send_partial_response(server_slot & slot, completion_token_output tkn) {- server_task_result res;- res.id = slot.id_task;- res.id_multi = slot.id_multi;- res.error = false;- res.stop = false;- res.data = json {+ res.result_json = json+ {{"content", tkn.text_to_send},{"stop", false},- {"id_slot", slot.id},- {"multimodal", false}+ {"slot_id", slot.id},+ {"multimodal", multimodal}};- if (slot.sparams.n_probs > 0) {+ if (slot.sparams.n_probs > 0)+ {+ std::vector<completion_token_output> probs_output = {};const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);- const size_t probs_pos = std::min(slot.n_sent_token_probs, slot.generated_token_probs.size());- const size_t probs_stop_pos = std::min(slot.n_sent_token_probs + to_send_toks.size(), slot.generated_token_probs.size());-- std::vector<completion_token_output> probs_output;- if (probs_pos < probs_stop_pos) {- probs_output = std::vector<completion_token_output>(- slot.generated_token_probs.begin() + probs_pos,- slot.generated_token_probs.begin() + probs_stop_pos);+ size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());+ size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size());+ if (probs_pos < probs_stop_pos)+ {+ probs_output = std::vector<completion_token_output>(slot.generated_token_probs.begin() + probs_pos, slot.generated_token_probs.begin() + probs_stop_pos);}- slot.n_sent_token_probs = probs_stop_pos;-- res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs_output);+ slot.sent_token_probs_index = probs_stop_pos;+ res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs_output);}- if (slot.oaicompat) {- res.data["oaicompat_token_ctr"] = slot.n_decoded;- res.data["model"] = slot.oaicompat_model;+ if (slot.oaicompat)+ {+ res.result_json["oaicompat_token_ctr"] = slot.n_decoded;+ res.result_json["model"] = slot.oaicompat_model;}queue_results.send(res);}- void send_final_response(const server_slot & slot) {- server_task_result res;- res.id = slot.id_task;- res.id_multi = slot.id_multi;- res.error = false;- res.stop = true;- res.data = json {+ void send_final_response(llama_client_slot &slot)+ {+ task_result res;+ res.id = slot.task_id;+ res.multitask_id = slot.multitask_id;+ res.error = false;+ res.stop = true;++ res.result_json = json+ {{"content", !slot.params.stream ? slot.generated_text : ""},- {"id_slot", slot.id},+ {"slot_id", slot.id},{"stop", true},{"model", params.model_alias},{"tokens_predicted", slot.n_decoded},- {"tokens_evaluated", slot.n_prompt_tokens},+ {"tokens_evaluated", slot.num_prompt_tokens},{"generation_settings", get_formated_generation(slot)},{"prompt", slot.prompt},{"truncated", slot.truncated},@@ -1338,91 +1239,80 @@ struct server_context {{"timings", slot.get_formated_timings()}};- if (slot.sparams.n_probs > 0) {- std::vector<completion_token_output> probs;- if (!slot.params.stream && slot.stopped_word) {+ if (slot.sparams.n_probs > 0)+ {+ std::vector<completion_token_output> probs = {};+ if (!slot.params.stream && slot.stopped_word)+ {const std::vector<llama_token> stop_word_toks = llama_tokenize(ctx, slot.stopping_word, false);-- probs = std::vector<completion_token_output>(- slot.generated_token_probs.begin(),- slot.generated_token_probs.end() - stop_word_toks.size());- } else {+ probs = std::vector<completion_token_output>(slot.generated_token_probs.begin(), slot.generated_token_probs.end() - stop_word_toks.size());+ }+ else+ {probs = std::vector<completion_token_output>(- slot.generated_token_probs.begin(),- slot.generated_token_probs.end());+ slot.generated_token_probs.begin(),+ slot.generated_token_probs.end());}-- res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs);+ res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);}- if (slot.oaicompat) {- res.data["oaicompat_token_ctr"] = slot.n_decoded;- res.data["model"] = slot.oaicompat_model;+ if (slot.oaicompat)+ {+ res.result_json["oaicompat_token_ctr"] = slot.n_decoded;+ res.result_json["model"] = slot.oaicompat_model;}queue_results.send(res);}- void send_embedding(const server_slot & slot, const llama_batch & batch) {- server_task_result res;- res.id = slot.id_task;- res.id_multi = slot.id_multi;- res.error = false;- res.stop = true;+ void send_embedding(llama_client_slot &slot)+ {+ task_result res;+ res.id = slot.task_id;+ res.multitask_id = slot.multitask_id;+ res.error = false;+ res.stop = true;const int n_embd = llama_n_embd(model);-- std::vector<float> embd_res(n_embd, 0.0f);-- for (int i = 0; i < batch.n_tokens; ++i) {- if (!batch.logits[i] || batch.seq_id[i][0] != slot.id + 1) {- continue;- }-- const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]);- if (embd == NULL) {- embd = llama_get_embeddings_ith(ctx, i);- }-- if (embd == NULL) {- LOG_ERROR("failed to get embeddings", {- {"token", batch.token [i]},- {"seq_id", batch.seq_id[i][0]}- });-- res.data = json {- {"embedding", std::vector<float>(n_embd, 0.0f)},- };-- continue;- }-- llama_embd_normalize(embd, embd_res.data(), n_embd);-- res.data = json {- {"embedding", embd_res},+ if (!params.embedding)+ {+ LOG_WARNING("embedding disabled", {+ {"params.embedding", params.embedding},+ });+ res.result_json = json+ {+ {"embedding", std::vector<float>(n_embd, 0.0f)},+ };+ }+ else+ {+ const float *data = llama_get_embeddings(ctx);+ std::vector<float> embedding(data, data + n_embd);+ res.result_json = json+ {+ {"embedding", embedding },};}-queue_results.send(res);}- void request_completion(int id_task, int id_multi, json data, bool infill, bool embedding) {- server_task task;- task.id = id_task;- task.id_multi = id_multi;- task.id_target = 0;- task.data = std::move(data);- task.infill = infill;- task.embedding = embedding;- task.type = SERVER_TASK_TYPE_COMPLETION;+ void request_completion(int task_id, json data, bool infill, bool embedding, int multitask_id)+ {+ task_server task;+ task.id = task_id;+ task.target_id = 0;+ task.data = std::move(data);+ task.infill_mode = infill;+ task.embedding_mode = embedding;+ task.type = TASK_TYPE_COMPLETION;+ task.multitask_id = multitask_id;// when a completion task's prompt array is not a singleton, we split it into multiple requests// otherwise, it's a single-prompt task, we actually queue it// if there's numbers in the prompt array it will be treated as an array of tokensif (task.data.count("prompt") != 0 && task.data.at("prompt").size() > 1) {bool numbers = false;- for (const auto & e : task.data.at("prompt")) {+ for (const auto& e : task.data.at("prompt")) {if (e.is_number()) {numbers = true;break;@@ -1437,23 +1327,96 @@ struct server_context {if (numbers) {queue_tasks.post(task);} else {- split_multiprompt_task(id_task, task);+ split_multiprompt_task(task_id, task);}} else {+ // an empty prompt can make slot become buggy+ if (task.data.contains("prompt") && task.data["prompt"].is_string() && task.data["prompt"].get<std::string>().empty()) {+ task.data["prompt"] = " "; // add a space so that we have one token+ }queue_tasks.post(task);}}- void request_cancel(int id_task) {- server_task task;- task.type = SERVER_TASK_TYPE_CANCEL;- task.id_target = id_task;+ // for multiple images processing+ bool ingest_images(llama_client_slot &slot, int n_batch)+ {+ int image_idx = 0;++ while (image_idx < (int) slot.images.size())+ {+ slot_image &img = slot.images[image_idx];++ // process prefix prompt+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)+ {+ const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));+ llama_batch batch_view = {+ n_tokens,+ batch.token + i,+ nullptr,+ batch.pos + i,+ batch.n_seq_id + i,+ batch.seq_id + i,+ batch.logits + i,+ 0, 0, 0, // unused+ };+ if (llama_decode(ctx, batch_view))+ {+ LOG_TEE("%s : failed to eval\n", __func__);+ return false;+ }+ }++ // process image with llm+ for (int i = 0; i < img.image_tokens; i += n_batch)+ {+ int n_eval = img.image_tokens - i;+ if (n_eval > n_batch)+ {+ n_eval = n_batch;+ }++ const int n_embd = llama_n_embd(model);+ llama_batch batch_img = { n_eval, nullptr, (img.image_embedding + i * n_embd), nullptr, nullptr, nullptr, nullptr, slot.n_past, 1, 0, };+ if (llama_decode(ctx, batch_img))+ {+ LOG_TEE("%s : failed to eval image\n", __func__);+ return false;+ }+ slot.n_past += n_eval;+ }+ image_idx++;++ llama_batch_clear(batch);++ // append prefix of next image+ const auto json_prompt = (image_idx >= (int) slot.images.size()) ?+ slot.params.input_suffix : // no more images, then process suffix prompt+ (json)(slot.images[image_idx].prefix_prompt);+ std::vector<llama_token> append_tokens = tokenize(json_prompt, false); // has next image+ for (int i = 0; i < (int) append_tokens.size(); ++i)+ {+ llama_batch_add(batch, append_tokens[i], system_tokens.size() + slot.n_past, { slot.id }, true);+ slot.n_past += 1;+ }+ }++ return true;+ }++ void request_cancel(int task_id)+ {+ task_server task;+ task.type = TASK_TYPE_CANCEL;+ task.target_id = task_id;queue_tasks.post(task);}- void split_multiprompt_task(int id_multi, const server_task & multiprompt_task) {- const int prompt_count = multiprompt_task.data.at("prompt").size();+ void split_multiprompt_task(int multitask_id, task_server& multiprompt_task)+ {+ int prompt_count = multiprompt_task.data.at("prompt").size();if (prompt_count <= 1) {send_error(multiprompt_task, "error while handling multiple prompts");return;@@ -1461,239 +1424,208 @@ struct server_context {// generate all the ID for subtaskstd::vector<int> subtask_ids(prompt_count);- for (int i = 0; i < prompt_count; i++) {+ for (int i = 0; i < prompt_count; i++)+ {subtask_ids[i] = queue_tasks.get_new_id();}// queue up the multitask so we can track its subtask progression- queue_tasks.add_multitask(id_multi, subtask_ids);+ queue_tasks.add_multitask(multitask_id, subtask_ids);// add subtasks- for (int i = 0; i < prompt_count; i++) {+ for (int i = 0; i < prompt_count; i++)+ {json subtask_data = multiprompt_task.data;subtask_data["prompt"] = subtask_data["prompt"][i];// subtasks inherit everything else (infill mode, embedding mode, etc.)- request_completion(subtask_ids[i], id_multi, subtask_data, multiprompt_task.infill, multiprompt_task.embedding);+ request_completion(subtask_ids[i], subtask_data, multiprompt_task.infill_mode, multiprompt_task.embedding_mode, multitask_id);}}- void process_single_task(const server_task & task) {- switch (task.type) {- case SERVER_TASK_TYPE_COMPLETION:+ void process_single_task(task_server& task)+ {+ switch (task.type)+ {+ case TASK_TYPE_COMPLETION: {+ llama_client_slot *slot = get_slot(json_value(task.data, "slot_id", -1));+ if (slot == nullptr)+ {+ // if no slot is available, we defer this task for processing later+ LOG_VERBOSE("no slot is available", {{"task_id", task.id}});+ queue_tasks.defer(task);+ break;+ }++ if (task.data.contains("system_prompt")){- server_slot * slot = get_slot(json_value(task.data, "id_slot", -1));- if (slot == nullptr) {- // if no slot is available, we defer this task for processing later- LOG_VERBOSE("no slot is available", {{"id_task", task.id}});- queue_tasks.defer(task);+ if (!all_slots_are_idle) {+ send_error(task, "system prompt can only be updated when all slots are idle");break;}+ process_system_prompt_data(task.data["system_prompt"]);- if (task.data.contains("system_prompt")) {- system_prompt_set(task.data["system_prompt"]);-- for (server_slot & slot : slots) {- slot.n_past = 0;- slot.n_past_se = 0;- }+ // reset cache_tokens for all slots+ for (llama_client_slot &slot : slots)+ {+ slot.cache_tokens.clear();+ slot.n_past = 0;+ slot.n_past_se = 0;}+ }- slot->reset();+ slot->reset();- slot->id_task = task.id;- slot->id_multi = task.id_multi;- slot->infill = task.infill;- slot->embedding = task.embedding;+ slot->infill = task.infill_mode;+ slot->embedding = task.embedding_mode;+ slot->task_id = task.id;+ slot->multitask_id = task.multitask_id;- if (!launch_slot_with_task(*slot, task)) {- LOG_ERROR("error while launching slot", task.data);- break;- }- } break;- case SERVER_TASK_TYPE_CANCEL:+ if (!launch_slot_with_data(slot, task.data)){- // release slot linked with the task id- for (auto & slot : slots) {- if (slot.id_task == task.id_target) {- slot.release();- break;- }- }- } break;- case SERVER_TASK_TYPE_NEXT_RESPONSE:- {- // do nothing- } break;- case SERVER_TASK_TYPE_METRICS:+ // send error result+ send_error(task, "internal_error");+ break;+ }+ } break;+ case TASK_TYPE_CANCEL: { // release slot linked with the task id+ for (auto & slot : slots){- json slots_data = json::array();-- int n_idle_slots = 0;- int n_processing_slots = 0;-- for (server_slot & slot : slots) {- json slot_data = get_formated_generation(slot);- slot_data["id"] = slot.id;- slot_data["id_task"] = slot.id_task;- slot_data["state"] = slot.state;- slot_data["prompt"] = slot.prompt;- slot_data["next_token"] = {+ if (slot.task_id == task.target_id)+ {+ slot.release();+ break;+ }+ }+ } break;+ case TASK_TYPE_NEXT_RESPONSE: {+ // do nothing+ } break;+ case TASK_TYPE_METRICS: {+ json slots_data = json::array();+ int n_idle_slots = 0;+ int n_processing_slots = 0;++ for (llama_client_slot &slot: slots) {+ json slot_data = get_formated_generation(slot);+ slot_data["id"] = slot.id;+ slot_data["task_id"] = slot.task_id;+ slot_data["state"] = slot.state;+ slot_data["prompt"] = slot.prompt;+ slot_data["next_token"] = {{"has_next_token", slot.has_next_token},- {"n_remain", slot.n_remaining},- {"n_decoded", slot.n_decoded},- {"stopped_eos", slot.stopped_eos},- {"stopped_word", slot.stopped_word},- {"stopped_limit", slot.stopped_limit},- {"stopping_word", slot.stopping_word},- };-- if (slot_data["state"] == SLOT_STATE_IDLE) {- n_idle_slots++;- } else {- n_processing_slots++;- }-- slots_data.push_back(slot_data);+ {"n_remain", slot.n_remaining},+ {"num_tokens_predicted", slot.n_decoded},+ {"stopped_eos", slot.stopped_eos},+ {"stopped_word", slot.stopped_word},+ {"stopped_limit", slot.stopped_limit},+ {"stopping_word", slot.stopping_word},+ };+ if (slot_data["state"] == IDLE) {+ n_idle_slots++;+ } else {+ n_processing_slots++;}- LOG_INFO("slot data", {- {"id_task", task.id},- {"n_idle_slots", n_idle_slots},- {"n_processing_slots", n_processing_slots}- });-- LOG_VERBOSE("slot data", {- {"id_task", task.id},- {"n_idle_slots", n_idle_slots},- {"n_processing_slots", n_processing_slots},- {"slots", slots_data}- });-- server_task_result res;- res.id = task.id;- res.id_multi = task.id_multi;- res.stop = true;- res.error = false;- res.data = {+ slots_data.push_back(slot_data);+ }+ LOG_INFO("slot data", {+ {"task_id", task.id},+ {"n_idle_slots", n_idle_slots},+ {"n_processing_slots", n_processing_slots}+ });+ LOG_VERBOSE("slot data", {+ {"task_id", task.id},+ {"n_idle_slots", n_idle_slots},+ {"n_processing_slots", n_processing_slots},+ {"slots", slots_data}+ });+ task_result res;+ res.id = task.id;+ res.multitask_id = task.multitask_id;+ res.stop = true;+ res.error = false;+ res.result_json = {{ "idle", n_idle_slots },{ "processing", n_processing_slots },{ "deferred", queue_tasks.queue_tasks_deferred.size() },- { "t_start", metrics.t_start},{ "n_prompt_tokens_processed_total", metrics.n_prompt_tokens_processed_total},- { "t_tokens_generation_total", metrics.t_tokens_generation_total},{ "n_tokens_predicted_total", metrics.n_tokens_predicted_total},- { "t_prompt_processing_total", metrics.t_prompt_processing_total},{ "n_prompt_tokens_processed", metrics.n_prompt_tokens_processed},{ "t_prompt_processing", metrics.t_prompt_processing},{ "n_tokens_predicted", metrics.n_tokens_predicted},{ "t_tokens_generation", metrics.t_tokens_generation},- { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)},- { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)},-- { "slots", slots_data },- };+ { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)},+ { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)},- if (json_value(task.data, "reset_bucket", false)) {- metrics.reset_bucket();- }- queue_results.send(res);- } break;+ { "slots", slots_data },+ };+ metrics.reset_bucket();+ queue_results.send(res);+ } break;}}- void on_finish_multitask(const server_task_multi & multitask) {+ void on_finish_multitask(task_multi& multitask)+ {// all subtasks done == multitask is done- server_task_result result;- result.id = multitask.id;- result.stop = true;+ task_result result;+ result.id = multitask.id;+ result.stop = true;result.error = false;// collect json results into one json resultstd::vector<json> result_jsons;- for (const auto & subres : multitask.results) {- result_jsons.push_back(subres.data);+ for (auto& subres : multitask.results)+ {+ result_jsons.push_back(subres.result_json);result.error = result.error && subres.error;}- result.data = json {- { "results", result_jsons }- };-+ result.result_json = json{ { "results", result_jsons } };queue_results.send(result);}- void update_slots() {- if (system_need_update) {- system_prompt_update();+ bool update_slots() {+ if (system_need_update)+ {+ LOG_INFO("updating system prompt", {});+ update_system_prompt();}- // release slots- for (auto & slot : slots) {- if (slot.command == SLOT_COMMAND_RELEASE) {- slot.state = SLOT_STATE_IDLE;- slot.command = SLOT_COMMAND_NONE;- slot.t_last_used = ggml_time_us();-- LOG_INFO("slot released", {- {"id_slot", slot.id},- {"id_task", slot.id_task},- {"n_ctx", n_ctx},- {"n_past", slot.n_past},- {"n_system_tokens", system_tokens.size()},- {"n_cache_tokens", slot.cache_tokens.size()},- {"truncated", slot.truncated}- });-- queue_tasks.notify_slot_changed();- }- }+ llama_batch_clear(batch);- // check if all slots are idle+ if (all_slots_are_idle){- bool all_idle = true;-- for (auto & slot : slots) {- if (slot.state != SLOT_STATE_IDLE || slot.command != SLOT_COMMAND_NONE) {- all_idle = false;- break;- }- }-- if (all_idle) {- LOG_INFO("all slots are idle", {});- if (system_prompt.empty() && clean_kv_cache) {- kv_cache_clear();- }-- return;+ if (system_prompt.empty() && clean_kv_cache)+ {+ LOG_INFO("all slots are idle and system prompt is empty, clear the KV cache", {});+ kv_cache_clear();}+ return true;}- {- LOG_VERBOSE("posting NEXT_RESPONSE", {});-- server_task task;- task.type = SERVER_TASK_TYPE_NEXT_RESPONSE;- task.id_target = -1;-- queue_tasks.post(task);- }+ LOG_VERBOSE("posting NEXT_RESPONSE", {});+ task_server task;+ task.type = TASK_TYPE_NEXT_RESPONSE;+ task.target_id = -1;+ queue_tasks.post(task);- // apply context-shift if needed- // TODO: simplify and improve- for (server_slot & slot : slots) {- if (slot.ga_n == 1) {- if (slot.is_processing() && (int) system_tokens.size() + slot.n_past >= slot.n_ctx - 1) {+ for (llama_client_slot &slot : slots)+ {+ if (slot.ga_n == 1)+ {+ if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx)+ {// Shift contextconst int n_keep = slot.params.n_keep + add_bos_token;const int n_left = (int) system_tokens.size() + slot.n_past - n_keep;const int n_discard = n_left / 2;LOG_INFO("slot context shift", {- {"id_slot", slot.id},- {"id_task", slot.id_task},+ {"slot_id", slot.id},+ {"task_id", slot.task_id},{"n_keep", n_keep},{"n_left", n_left},{"n_discard", n_discard},@@ -1702,18 +1634,16 @@ struct server_context {{"n_system_tokens", system_tokens.size()},{"n_cache_tokens", slot.cache_tokens.size()}});+ llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard);+ llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);- llama_kv_cache_seq_rm (ctx, slot.id + 1, n_keep , n_keep + n_discard);- llama_kv_cache_seq_add(ctx, slot.id + 1, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);-- if (slot.params.cache_prompt) {- for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) {- slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];- }-- slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard);+ for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++)+ {+ slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];}+ slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard);+slot.n_past -= n_discard;slot.truncated = true;@@ -1721,12 +1651,33 @@ struct server_context {}}- // start populating the batch for this iteration- llama_batch_clear(batch);+ // decode any currently ongoing sequences+ LOG_VERBOSE("decoding ongoing sequences", {});+ for (auto & slot : slots)+ {+ // release the slot+ if (slot.command == RELEASE)+ {+ slot.state = IDLE;+ slot.command = NONE;+ slot.t_last_used = ggml_time_us();++ LOG_INFO("slot released", {+ {"slot_id", slot.id},+ {"task_id", slot.task_id},+ {"n_ctx", n_ctx},+ {"n_past", slot.n_past},+ {"n_system_tokens", system_tokens.size()},+ {"n_cache_tokens", slot.cache_tokens.size()},+ {"truncated", slot.truncated}+ });+ queue_tasks.notify_slot_changed();++ continue;+ }- // frist, add sampled tokens from any ongoing sequences- for (auto & slot : slots) {- if (slot.state == SLOT_STATE_IDLE) {+ if (slot.state == IDLE)+ {continue;}@@ -1736,217 +1687,187 @@ struct server_context {// TODO: we always have to take into account the "system_tokens"// this is not great and needs to be improved somehow- llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id + 1 }, true);-+ llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true);slot.n_past += 1;-- if (slot.params.cache_prompt) {- slot.cache_tokens.push_back(slot.sampled);- }-- LOG_VERBOSE("slot decode token", {- {"id_slot", slot.id},- {"id_task", slot.id_task},- {"n_ctx", n_ctx},- {"n_past", slot.n_past},- {"n_system_tokens", system_tokens.size()},- {"n_cache_tokens", slot.cache_tokens.size()},- {"truncated", slot.truncated}- });}// process in chunks of params.n_batch- int32_t n_batch = llama_n_batch(ctx);- int32_t n_ubatch = llama_n_ubatch(ctx);-- // next, batch any pending prompts without exceeding n_batch- if (params.cont_batching || batch.n_tokens == 0) {- for (auto & slot : slots) {- // this slot still has a prompt to be processed- if (slot.state == SLOT_STATE_IDLE && slot.command == SLOT_COMMAND_LOAD_PROMPT) {- auto & prompt_tokens = slot.prompt_tokens;-- // we haven't tokenized the prompt yet - do it now:- if (prompt_tokens.empty()) {- LOG_VERBOSE("tokenizing prompt", {- {"id_slot", slot.id},- {"id_task", slot.id_task}- });-- slot.t_start_process_prompt = ggml_time_us();- slot.t_start_generation = 0;+ int32_t n_batch = params.n_batch;- if (slot.infill) {- bool suff_rm_leading_spc = true;- if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1) {- params.input_suffix.erase(0, 1);- suff_rm_leading_spc = false;- }-- auto prefix_tokens = tokenize(slot.params.input_prefix, false);- auto suffix_tokens = tokenize(slot.params.input_suffix, false);+ // assign workload to the slots+ if (params.cont_batching || batch.n_tokens == 0)+ {+ for (auto & slot : slots)+ {+ const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get<std::string>().empty()) || !slot.images.empty();- const int space_token = 29871; // TODO: this should not be hardcoded- if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {- suffix_tokens.erase(suffix_tokens.begin());- }+ // empty prompt passed -> release the slot and send empty response+ // note: infill mode allows empty prompt+ if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt && !slot.infill)+ {+ slot.release();+ slot.print_timings();+ send_final_response(slot);+ continue;+ }- prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));- prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS- prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));- prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());- prefix_tokens.push_back(llama_token_middle(model));- prompt_tokens = prefix_tokens;- } else {- prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt+ // need process the prompt+ if (slot.state == IDLE && slot.command == LOAD_PROMPT)+ {+ slot.state = PROCESSING;+ slot.command = NONE;+ std::vector<llama_token> prompt_tokens;+ slot.t_start_process_prompt = ggml_time_us();+ slot.t_start_genereration = 0;++ if (slot.infill)+ {+ bool suff_rm_leading_spc = true;+ if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1)+ {+ params.input_suffix.erase(0, 1);+ suff_rm_leading_spc = false;}+ auto prefix_tokens = tokenize(slot.params.input_prefix, false);+ auto suffix_tokens = tokenize(slot.params.input_suffix, false);- slot.n_past = 0;- slot.n_prompt_tokens = prompt_tokens.size();-- LOG_VERBOSE("prompt tokenized", {- {"id_slot", slot.id},- {"id_task", slot.id_task},- {"n_ctx", slot.n_ctx},- {"n_keep", slot.params.n_keep},- {"n_prompt_tokens", slot.n_prompt_tokens},- {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())},- });-- // empty prompt passed -> release the slot and send empty response- if (prompt_tokens.empty()) {- LOG_INFO("empty prompt - releasing slot", {- {"id_slot", slot.id},- {"id_task", slot.id_task}- });-- slot.state = SLOT_STATE_PROCESSING;- slot.command = SLOT_COMMAND_NONE;- slot.release();- slot.print_timings();- send_final_response(slot);- continue;+ const int space_token = 29871; // TODO: this should not be hardcoded+ if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {+ suffix_tokens.erase(suffix_tokens.begin());}- if (slot.embedding) {- // this prompt is too large to process - discard it- if (slot.n_prompt_tokens > n_ubatch) {- slot.state = SLOT_STATE_PROCESSING;- slot.command = SLOT_COMMAND_NONE;- slot.release();- slot.print_timings();- send_final_response(slot);- continue;- }- } else {- if (slot.params.n_keep < 0) {- slot.params.n_keep = slot.n_prompt_tokens;- }- slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep);-- // if input prompt is too big, truncate it (if group attention self-extend is disabled)- if (slot.ga_n == 1 && slot.n_prompt_tokens >= slot.n_ctx) {- const int n_left = slot.n_ctx - slot.params.n_keep;-- const int n_block_size = n_left / 2;- const int erased_blocks = (slot.n_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size;-- std::vector<llama_token> new_tokens(- prompt_tokens.begin(),- prompt_tokens.begin() + slot.params.n_keep);-- new_tokens.insert(- new_tokens.end(),- prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size,- prompt_tokens.end());-- prompt_tokens = std::move(new_tokens);+ prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));+ prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS+ prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));+ prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());+ prefix_tokens.push_back(llama_token_middle(model));+ prompt_tokens = prefix_tokens;+ }+ else+ {+ prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt+ }- slot.truncated = true;- slot.n_prompt_tokens = prompt_tokens.size();+ slot.num_prompt_tokens = prompt_tokens.size();- LOG_VERBOSE("input truncated", {- {"id_slot", slot.id},- {"id_task", slot.id_task},- {"n_ctx", slot.n_ctx},- {"n_keep", slot.params.n_keep},- {"n_left", n_left},- {"n_prompt_tokens", slot.n_prompt_tokens},- {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())},- });+ if (slot.params.n_keep < 0)+ {+ slot.params.n_keep = slot.num_prompt_tokens;+ }+ slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep);++ // if input prompt is too big, truncate it+ if (slot.num_prompt_tokens >= slot.n_ctx)+ {+ const int n_left = slot.n_ctx - slot.params.n_keep;+ const int n_block_size = n_left / 2;+ const int erased_blocks = (slot.num_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size;++ std::vector<llama_token> new_tokens(prompt_tokens.begin(), prompt_tokens.begin() + slot.params.n_keep);+ new_tokens.insert(new_tokens.end(), prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size, prompt_tokens.end());++ LOG_VERBOSE("input truncated", {+ {"n_ctx", slot.n_ctx},+ {"n_keep", slot.params.n_keep},+ {"n_left", n_left},+ {"new_tokens", tokens_to_str(ctx, new_tokens.cbegin(), new_tokens.cend())},+ });+ slot.truncated = true;+ prompt_tokens = new_tokens;- GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx);- }+ slot.num_prompt_tokens = prompt_tokens.size();+ GGML_ASSERT(slot.num_prompt_tokens < slot.n_ctx);+ }- llama_sampling_reset(slot.ctx_sampling);+ if (!slot.params.cache_prompt)+ {+ llama_sampling_reset(slot.ctx_sampling);- if (!slot.params.cache_prompt) {- slot.n_past_se = 0;- slot.ga_i = 0;- } else {- GGML_ASSERT(slot.ga_n == 1);+ slot.n_past = 0;+ slot.n_past_se = 0;+ slot.ga_i = 0;+ slot.num_prompt_tokens_processed = slot.num_prompt_tokens;+ }+ else+ {+ // push the prompt into the sampling context (do not apply grammar)+ for (auto &token : prompt_tokens)+ {+ llama_sampling_accept(slot.ctx_sampling, ctx, token, false);+ }- // reuse any previously computed tokens that are common with the new prompt- slot.n_past = common_part(slot.cache_tokens, prompt_tokens);+ slot.n_past = common_part(slot.cache_tokens, prompt_tokens);- // push the prompt into the sampling context (do not apply grammar)- for (int i = 0; i < slot.n_past; ++i) {- llama_sampling_accept(slot.ctx_sampling, ctx, slot.cache_tokens[i], false);- }- }+ // the last token of the cache is not in the KV cache until the next call to llama_decode+ // (it was sampled, pushed into the "cache_tokens", but not yet put in the context)+ if (slot.n_past > 0 && slot.n_past == (int32_t) slot.cache_tokens.size())+ {+ slot.n_past -= 1;}- if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0) {- // we have to evaluate at least 1 token to generate logits.- LOG_INFO("we have to evaluate at least 1 token to generate logits", {- { "id_slot", slot.id },- { "id_task", slot.id_task }- });-- slot.n_past--;- if (slot.ga_i > 0) {- slot.n_past_se--;+ slot.num_prompt_tokens_processed = slot.num_prompt_tokens - slot.n_past;++ if (slot.ga_n != 1)+ {+ int ga_i = 0;+ int32_t ga_n = slot.ga_n;+ int32_t ga_w = slot.ga_w;+ int32_t slot_npast = 0;+ for (int k = 0; k < slot.n_past; ++k)+ {+ while (slot_npast >= ga_i + ga_w) {+ const int bd = (ga_w/ga_n)*(ga_n - 1);+ slot_npast -= bd;+ ga_i += ga_w/ga_n;+ }+ slot_npast++;}+ slot.n_past_se = slot_npast;+ slot.ga_i = ga_i;}- slot.n_prompt_tokens_processed = 0;+ LOG_INFO("slot progression", {+ { "slot_id", slot.id },+ { "task_id", slot.task_id },+ { "n_past", slot.n_past },+ { "num_prompt_tokens_processed", slot.num_prompt_tokens_processed }+ });}- if (slot.embedding) {- // cannot fit the prompt in the current batch - will try next iter- if (batch.n_tokens + slot.n_prompt_tokens > n_batch) {- continue;- }- }+ slot.cache_tokens = prompt_tokens;- // keep only the common part- int p0 = (int) system_tokens.size() + slot.n_past;- if (!llama_kv_cache_seq_rm(ctx, slot.id + 1, p0, -1)) {- // could not partially delete (likely using a non-Transformer model)- llama_kv_cache_seq_rm(ctx, slot.id + 1, -1, -1);-- p0 = (int) system_tokens.size();- if (p0 != 0) {- // copy over the system prompt when there is one- llama_kv_cache_seq_cp(ctx, 0, slot.id + 1, -1, -1);+ if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0)+ {+ // we have to evaluate at least 1 token to generate logits.+ LOG_INFO("we have to evaluate at least 1 token to generate logits", {+ { "slot_id", slot.id },+ { "task_id", slot.task_id }+ });+ slot.n_past--;+ if (slot.ga_i > 0)+ {+ slot.n_past_se--;}-- // there is no common part left (except for the system prompt)- slot.n_past = 0;- slot.n_past_se = 0;- slot.ga_i = 0;- // TODO: is the system prompt ever in the sampling context?- llama_sampling_reset(slot.ctx_sampling);}- // remove the non-common part from the cache- slot.cache_tokens.resize(slot.n_past);-+ int p0 = (int) system_tokens.size() + slot.n_past;LOG_INFO("kv cache rm [p0, end)", {- { "id_slot", slot.id },- { "id_task", slot.id_task },+ { "slot_id", slot.id },+ { "task_id", slot.task_id },{ "p0", p0 }});+ llama_kv_cache_seq_rm(ctx, slot.id, p0, -1);++ LOG_VERBOSE("prompt ingested", {+ {"n_past", slot.n_past},+ {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},+ {"to_eval", tokens_to_str(ctx, slot.cache_tokens.cbegin() + slot.n_past, slot.cache_tokens.cend())},+ });++ const bool has_images = process_images(slot);++ // process the prefix of first image+ std::vector<llama_token> prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens;int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past;@@ -1954,81 +1875,61 @@ struct server_context {int32_t ga_n = slot.ga_n;int32_t ga_w = slot.ga_w;- // add prompt tokens for processing in the current batch- // TODO: the self-extend stuff here is a mess - simplify and/or abstract it somehow- for (; slot.n_past < slot.n_prompt_tokens && batch.n_tokens < n_batch; ++slot.n_past) {- if (slot.ga_n != 1) {+ for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past)+ {+ if (slot.ga_n != 1)+ {while (slot_npast >= ga_i + ga_w) {const int bd = (ga_w/ga_n)*(ga_n - 1);slot_npast -= bd;ga_i += ga_w/ga_n;}}-- llama_batch_add(batch, prompt_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id + 1 }, false);-- if (slot.params.cache_prompt) {- slot.cache_tokens.push_back(prompt_tokens[slot.n_past]);- }-- slot.n_prompt_tokens_processed++;+ llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, {slot.id }, false);slot_npast++;}- LOG_VERBOSE("prompt processing progress", {- {"id_slot", slot.id},- {"n_past", slot.n_past},- {"n_ctx", n_ctx},- {"n_tokens", batch.n_tokens},- {"progress", (float) slot.n_prompt_tokens_processed / slot.n_prompt_tokens},- });-- // entire prompt has been processed - start decoding new tokens- if (slot.n_past == slot.n_prompt_tokens) {- slot.state = SLOT_STATE_PROCESSING;- slot.command = SLOT_COMMAND_NONE;-- GGML_ASSERT(batch.n_tokens > 0);+ if (has_images && !ingest_images(slot, n_batch))+ {+ LOG_ERROR("failed processing images", {+ "slot_id", slot.id,+ "task_id", slot.task_id,+ });+ // FIXME @phymbert: to be properly tested+ // early returning without changing the slot state will block the slot for ever+ // no one at the moment is checking the return value+ return false;+ }- // extract the logits only for the last token+ // extract the logits only for the last token+ if (batch.n_tokens > 0)+ {batch.logits[batch.n_tokens - 1] = true;-- slot.n_decoded = 0;- slot.i_batch = batch.n_tokens - 1;-- LOG_VERBOSE("prompt done", {- {"id_slot", slot.id},- {"n_past", slot.n_past},- {"n_ctx", n_ctx},- {"n_tokens", batch.n_tokens},- });}- }- if (batch.n_tokens >= n_batch) {- break;+ slot.n_decoded = 0;+ slot.i_batch = batch.n_tokens - 1;}}}- if (batch.n_tokens == 0) {- LOG_VERBOSE("no tokens to decode", {});- return;+ if (batch.n_tokens == 0)+ {+ all_slots_are_idle = true;+ return true;}- LOG_VERBOSE("decoding batch", {- {"n_tokens", batch.n_tokens},- });-- // process the created batch of tokens- for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) {- const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)+ {+ const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));- for (auto & slot : slots) {- if (slot.ga_n != 1) {+ for (auto & slot : slots)+ {+ if (slot.ga_n != 1)+ {// context extension via Self-Extend- // TODO: simplify and/or abstract this- while (slot.n_past_se >= slot.ga_i + slot.ga_w) {+ while (slot.n_past_se >= slot.ga_i + slot.ga_w)+ {const int ib = (slot.ga_n * slot.ga_i) / slot.ga_w;const int bd = (slot.ga_w / slot.ga_n) * (slot.ga_n - 1);const int dd = (slot.ga_w / slot.ga_n) - ib * bd - slot.ga_w;@@ -2038,9 +1939,9 @@ struct server_context {LOG_TEE("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n, (slot.ga_i + ib * bd) / slot.ga_n, (slot.ga_i + ib * bd + slot.ga_w) / slot.ga_n);LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd, slot.ga_i + ib * bd + slot.ga_w + dd, slot.n_past_se + ib * bd + dd);- llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i, slot.n_past_se, ib * bd);- llama_kv_cache_seq_div(ctx, slot.id + 1, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n);- llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd);+ llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd);+ llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w,slot.ga_n);+ llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w,slot.n_past_se + ib * bd, dd);slot.n_past_se -= bd;@@ -2048,12 +1949,12 @@ struct server_context {LOG_TEE("\nn_past_old = %d, n_past = %d, ga_i = %d\n\n", slot.n_past_se + bd, slot.n_past_se, slot.ga_i);}-slot.n_past_se += n_tokens;}}- llama_batch batch_view = {+ llama_batch batch_view =+ {n_tokens,batch.token + i,nullptr,@@ -2066,17 +1967,13 @@ struct server_context {const int ret = llama_decode(ctx, batch_view);- if (ret != 0) {- if (n_batch == 1 || ret < 0) {+ if (ret != 0)+ {+ if (n_batch == 1 || ret < 0)+ {// if you get here, it means the KV cache is full - try increasing it via the context sizeLOG_TEE("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret);- for (auto & slot : slots) {- slot.state = SLOT_STATE_PROCESSING;- slot.command = SLOT_COMMAND_NONE;- slot.release();- send_error(slot, "Input prompt is too big compared to KV size. Please try increasing KV size.");- }- break; // break loop of n_batch+ return false;}LOG_TEE("%s : failed to find free space in the KV cache, retrying with smaller n_batch = %d\n", __func__, n_batch / 2);@@ -2084,21 +1981,23 @@ struct server_context {// retry with half the batch size to try to find a free slot in the KV cachen_batch /= 2;i -= n_batch;-- continue; // continue loop of n_batch+ continue;}- for (auto & slot : slots) {- if (slot.state != SLOT_STATE_PROCESSING || slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) {- continue; // continue loop of slots+ for (auto & slot : slots)+ {+ if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens))+ {+ continue;}// prompt evaluated for embedding- if (slot.embedding) {- send_embedding(slot, batch_view);+ if (slot.embedding)+ {+ send_embedding(slot);slot.release();slot.i_batch = -1;- continue; // continue loop of slots+ continue;}completion_token_output result;@@ -2107,9 +2006,10 @@ struct server_context {llama_sampling_accept(slot.ctx_sampling, ctx, id, true);slot.n_decoded += 1;- if (slot.n_decoded == 1) {- slot.t_start_generation = ggml_time_us();- slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3;+ if (slot.n_decoded == 1)+ {+ slot.t_start_genereration = ggml_time_us();+ slot.t_prompt_processing = (slot.t_start_genereration - slot.t_start_process_prompt) / 1e3;metrics.on_prompt_eval(slot);}@@ -2117,19 +2017,19 @@ struct server_context {result.tok = id;const int32_t n_probs = slot.sparams.n_probs;- if (slot.sparams.temp <= 0 && n_probs > 0) {+ if (slot.sparams.temp <= 0 && n_probs > 0)+ {// for llama_sample_token_greedy we need to sort candidatesllama_sample_softmax(ctx, &cur_p);}- for (size_t i = 0; i < std::min(cur_p.size, (size_t) n_probs); ++i) {- result.probs.push_back({- cur_p.data[i].id,- cur_p.data[i].p- });+ for (size_t i = 0; i < std::min(cur_p.size, (size_t)n_probs); ++i)+ {+ result.probs.push_back({cur_p.data[i].id, cur_p.data[i].p});}- if (!process_token(result, slot)) {+ if (!process_token(result, slot))+ {slot.release();slot.print_timings();send_final_response(slot);@@ -2140,22 +2040,18 @@ struct server_context {}}- LOG_VERBOSE("run slots completed", {});+ LOG_VERBOSE("slots updated", {});+ return true;}- json model_meta() const {- return json {- {"vocab_type", llama_vocab_type (model)},- {"n_vocab", llama_n_vocab (model)},- {"n_ctx_train", llama_n_ctx_train (model)},- {"n_embd", llama_n_embd (model)},- {"n_params", llama_model_n_params(model)},- {"size", llama_model_size (model)},- };+ void run_on_all_tasks_finished() {+ update_slots();}};-static void server_print_usage(const char * argv0, const gpt_params & params, const server_params & sparams) {+static void server_print_usage(const char *argv0, const gpt_params ¶ms,+ const server_params &sparams)+{printf("usage: %s [options]\n", argv0);printf("\n");printf("options:\n");@@ -2163,7 +2059,6 @@ static void server_print_usage(const char * argv0, const gpt_params & params, coprintf(" -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled");printf(" -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);printf(" -tb N, --threads-batch N number of threads to use during batch and prompt processing (default: same as --threads)\n");- printf(" --threads-http N number of threads in the http server pool to process requests (default: max(hardware concurrency - 1, --parallel N + 2))\n");printf(" -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx);printf(" --rope-scaling {none,linear,yarn}\n");printf(" RoPE frequency scaling method, defaults to linear unless specified by the model\n");@@ -2173,17 +2068,15 @@ static void server_print_usage(const char * argv0, const gpt_params & params, coprintf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n");printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow);printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast);- printf(" --pooling {none,mean,cls} pooling type for embeddings, use model default if unspecified\n");- printf(" -dt N, --defrag-thold N\n");- printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold);- printf(" -b N, --batch-size N logical maximum batch size (default: %d)\n", params.n_batch);- printf(" -ub N, --ubatch-size N physical maximum batch size (default: %d)\n", params.n_ubatch);+ printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n");printf(" not recommended: doubles context memory required and no measurable increase in quality\n");- if (llama_supports_mlock()) {+ if (llama_supports_mlock())+ {printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");}- if (llama_supports_mmap()) {+ if (llama_supports_mmap())+ {printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");}printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n");@@ -2205,23 +2098,17 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co}printf(" -m FNAME, --model FNAME\n");printf(" model path (default: %s)\n", params.model.c_str());- printf(" -mu MODEL_URL, --model-url MODEL_URL\n");- printf(" model download url (default: %s)\n", params.model_url.c_str());printf(" -a ALIAS, --alias ALIAS\n");printf(" set an alias for the model, will be added as `model` field in completion response\n");printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);- printf(" --path PUBLIC_PATH path from which to serve static files (default: disabled)\n");+ printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- printf(" --ssl-key-file FNAME path to file a PEM-encoded SSL private key\n");- printf(" --ssl-cert-file FNAME path to file a PEM-encoded SSL certificate\n");-#endifprintf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);- printf(" --embeddings enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");+ printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");printf(" -spf FNAME, --system-prompt-file FNAME\n");@@ -2230,6 +2117,7 @@ static void server_print_usage(const char * argv0, const gpt_params & params, coprintf(" KV cache data type for K (default: f16)\n");printf(" -ctv TYPE, --cache-type-v TYPE\n");printf(" KV cache data type for V (default: f16)\n");+ printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");printf(" --log-format log output format: json or text (default: json)\n");printf(" --log-disable disables logging to a file.\n");printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n");@@ -2239,50 +2127,65 @@ static void server_print_usage(const char * argv0, const gpt_params & params, coprintf(" --override-kv KEY=TYPE:VALUE\n");printf(" advanced option to override model metadata by key. may be specified multiple times.\n");printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");- printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`\n");- printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`\n");+ printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`");+ printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`");printf(" --chat-template JINJA_TEMPLATE\n");printf(" set custom jinja chat template (default: template taken from model's metadata)\n");- printf(" only commonly used templates are accepted:\n");- printf(" https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template\n");+ printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n");printf("\n");}-static void server_params_parse(int argc, char ** argv, server_params & sparams, gpt_params & params) {- gpt_params default_params;+static void server_params_parse(int argc, char **argv, server_params &sparams,+ gpt_params ¶ms, llama_server_context& llama)+{+ gpt_params default_params;server_params default_sparams;-std::string arg;bool invalid_param = false;- for (int i = 1; i < argc; i++) {+ for (int i = 1; i < argc; i++)+ {arg = argv[i];- if (arg == "--port") {- if (++i >= argc) {+ if (arg == "--port")+ {+ if (++i >= argc)+ {invalid_param = true;break;}sparams.port = std::stoi(argv[i]);- } else if (arg == "--host") {- if (++i >= argc) {+ }+ else if (arg == "--host")+ {+ if (++i >= argc)+ {invalid_param = true;break;}sparams.hostname = argv[i];- } else if (arg == "--path") {- if (++i >= argc) {+ }+ else if (arg == "--path")+ {+ if (++i >= argc)+ {invalid_param = true;break;}sparams.public_path = argv[i];- } else if (arg == "--api-key") {- if (++i >= argc) {+ }+ else if (arg == "--api-key")+ {+ if (++i >= argc)+ {invalid_param = true;break;}- sparams.api_keys.push_back(argv[i]);- } else if (arg == "--api-key-file") {- if (++i >= argc) {+ sparams.api_keys.emplace_back(argv[i]);+ }+ else if (arg == "--api-key-file")+ {+ if (++i >= argc)+ {invalid_param = true;break;}@@ -2299,59 +2202,53 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,}}key_file.close();-- }-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- else if (arg == "--ssl-key-file") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.ssl_key_file = argv[i];- } else if (arg == "--ssl-cert-file") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.ssl_cert_file = argv[i];}-#endif- else if (arg == "--timeout" || arg == "-to") {- if (++i >= argc) {+ else if (arg == "--timeout" || arg == "-to")+ {+ if (++i >= argc)+ {invalid_param = true;break;}sparams.read_timeout = std::stoi(argv[i]);sparams.write_timeout = std::stoi(argv[i]);- } else if (arg == "-m" || arg == "--model") {- if (++i >= argc) {+ }+ else if (arg == "-m" || arg == "--model")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.model = argv[i];- } else if (arg == "-mu" || arg == "--model-url") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.model_url = argv[i];- } else if (arg == "-a" || arg == "--alias") {- if (++i >= argc) {+ }+ else if (arg == "-a" || arg == "--alias")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.model_alias = argv[i];- } else if (arg == "-h" || arg == "--help") {+ }+ else if (arg == "-h" || arg == "--help")+ {server_print_usage(argv[0], default_params, default_sparams);exit(0);- } else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size") {- if (++i >= argc) {+ }+ else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.n_ctx = std::stoi(argv[i]);- } else if (arg == "--rope-scaling") {- if (++i >= argc) {+ }+ else if (arg == "--rope-scaling")+ {+ if (++i >= argc)+ {invalid_param = true;break;}@@ -2360,142 +2257,150 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }else { invalid_param = true; break; }- } else if (arg == "--rope-freq-base") {- if (++i >= argc) {+ }+ else if (arg == "--rope-freq-base")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.rope_freq_base = std::stof(argv[i]);- } else if (arg == "--rope-freq-scale") {- if (++i >= argc) {+ }+ else if (arg == "--rope-freq-scale")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.rope_freq_scale = std::stof(argv[i]);- } else if (arg == "--yarn-ext-factor") {+ }+ else if (arg == "--yarn-ext-factor")+ {if (++i >= argc) {invalid_param = true;break;}params.yarn_ext_factor = std::stof(argv[i]);}- else if (arg == "--yarn-attn-factor") {+ else if (arg == "--yarn-attn-factor")+ {if (++i >= argc) {invalid_param = true;break;}params.yarn_attn_factor = std::stof(argv[i]);- } else if (arg == "--yarn-beta-fast") {+ }+ else if (arg == "--yarn-beta-fast")+ {if (++i >= argc) {invalid_param = true;break;}params.yarn_beta_fast = std::stof(argv[i]);- } else if (arg == "--yarn-beta-slow") {+ }+ else if (arg == "--yarn-beta-slow")+ {if (++i >= argc) {invalid_param = true;break;}params.yarn_beta_slow = std::stof(argv[i]);- } else if (arg == "--pooling") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::string value(argv[i]);- /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }- else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }- else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }- else { invalid_param = true; break; }- } else if (arg == "--defrag-thold" || arg == "-dt") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.defrag_thold = std::stof(argv[i]);- } else if (arg == "--threads" || arg == "-t") {+ }+ else if (arg == "--threads" || arg == "-t")+ {if (++i >= argc){invalid_param = true;break;}params.n_threads = std::stoi(argv[i]);- } else if (arg == "--grp-attn-n" || arg == "-gan") {+ }+ else if (arg == "--grp-attn-n" || arg == "-gan")+ {if (++i >= argc) {invalid_param = true;break;}params.grp_attn_n = std::stoi(argv[i]);- } else if (arg == "--grp-attn-w" || arg == "-gaw") {- if (++i >= argc) {+ }+ else if (arg == "--grp-attn-w" || arg == "-gaw")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.grp_attn_w = std::stoi(argv[i]);- } else if (arg == "--threads-batch" || arg == "-tb") {- if (++i >= argc) {+ }+ else if (arg == "--threads-batch" || arg == "-tb")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.n_threads_batch = std::stoi(argv[i]);- } else if (arg == "--threads-http") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.n_threads_http = std::stoi(argv[i]);- } else if (arg == "-b" || arg == "--batch-size") {- if (++i >= argc) {+ }+ else if (arg == "-b" || arg == "--batch-size")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.n_batch = std::stoi(argv[i]);- } else if (arg == "-ub" || arg == "--ubatch-size") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_ubatch = std::stoi(argv[i]);- } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {- if (++i >= argc) {+ params.n_batch = std::min(512, params.n_batch);+ }+ else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers")+ {+ if (++i >= argc)+ {invalid_param = true;break;}if (llama_supports_gpu_offload()) {params.n_gpu_layers = std::stoi(argv[i]);} else {- LOG_WARNING(- "Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "- "See main README.md for information on enabling GPU BLAS support",- {{"n_gpu_layers", params.n_gpu_layers}});+ LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "+ "See main README.md for information on enabling GPU BLAS support",+ {{"n_gpu_layers", params.n_gpu_layers}});}- } else if (arg == "--split-mode" || arg == "-sm") {+ }+ else if (arg == "--split-mode" || arg == "-sm")+ {if (++i >= argc) {invalid_param = true;break;}std::string arg_next = argv[i];- if (arg_next == "none") {+ if (arg_next == "none")+ {params.split_mode = LLAMA_SPLIT_MODE_NONE;- } else if (arg_next == "layer") {+ }+ else if (arg_next == "layer")+ {params.split_mode = LLAMA_SPLIT_MODE_LAYER;- } else if (arg_next == "row") {+ }+ else if (arg_next == "row")+ {params.split_mode = LLAMA_SPLIT_MODE_ROW;- } else {+ }+ else {invalid_param = true;break;}-#ifndef GGML_USE_CUBLAS- fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting the split mode has no effect.\n");-#endif // GGML_USE_CUBLAS- } else if (arg == "--tensor-split" || arg == "-ts") {- if (++i >= argc) {+ }+ else if (arg == "--tensor-split" || arg == "-ts")+ {+ if (++i >= argc)+ {invalid_param = true;break;}-#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL)std::string arg_next = argv[i];// split string by , and /@@ -2504,67 +2409,91 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,std::vector<std::string> split_arg{it, {}};GGML_ASSERT(split_arg.size() <= llama_max_devices());- for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device) {- if (i_device < split_arg.size()) {+ for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device)+ {+ if (i_device < split_arg.size())+ {params.tensor_split[i_device] = std::stof(split_arg[i_device]);- } else {+ }+ else+ {params.tensor_split[i_device] = 0.0f;}}-#else- LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a tensor split.\n", {});-#endif // GGML_USE_CUBLAS- } else if (arg == "--main-gpu" || arg == "-mg") {- if (++i >= argc) {+ }+ else if (arg == "--main-gpu" || arg == "-mg")+ {+ if (++i >= argc)+ {invalid_param = true;break;}-#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL)params.main_gpu = std::stoi(argv[i]);-#else- LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a main GPU.", {});-#endif- } else if (arg == "--lora") {- if (++i >= argc) {+ }+ else if (arg == "--lora")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.lora_adapter.emplace_back(argv[i], 1.0f);params.use_mmap = false;- } else if (arg == "--lora-scaled") {- if (++i >= argc) {+ }+ else if (arg == "--lora-scaled")+ {+ if (++i >= argc)+ {invalid_param = true;break;}const char * lora_adapter = argv[i];- if (++i >= argc) {+ if (++i >= argc)+ {invalid_param = true;break;}params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));params.use_mmap = false;- } else if (arg == "--lora-base") {- if (++i >= argc) {+ }+ else if (arg == "--lora-base")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.lora_base = argv[i];- } else if (arg == "-v" || arg == "--verbose") {+ }+ else if (arg == "-v" || arg == "--verbose")+ {#if SERVER_VERBOSE != 1LOG_WARNING("server.cpp is not built with verbose logging.", {});#elseserver_verbose = true;#endif- } else if (arg == "--nobrowser") {+ }+ else if (arg == "--nobrowser")+ {sparams.nobrowser = true;- } else if (arg == "--unsecure") {+ }+ else if (arg == "--unsecure")+ {FLAG_unsecure = true;- } else if (arg == "--server") {- } else if (arg == "--nocompile") {+ }+ else if (arg == "--server")+ {+ }+ else if (arg == "--nocompile")+ {FLAG_nocompile = true;- } else if (arg == "--recompile") {+ }+ else if (arg == "--recompile")+ {FLAG_recompile = true;- } else if (arg == "--gpu") {+ }+ else if (arg == "--gpu")+ {if (++i >= argc){invalid_param = true;@@ -2576,39 +2505,55 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);exit(1);}- } else if (arg == "--mlock") {+ }+ else if (arg == "--mlock")+ {params.use_mlock = true;- } else if (arg == "--no-mmap") {+ }+ else if (arg == "--no-mmap")+ {params.use_mmap = false;- } else if (arg == "--numa") {+ }+ else if (arg == "--numa") {if (++i >= argc) {invalid_param = true;break;} else {std::string value(argv[i]);/**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }- else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }- else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }else { invalid_param = true; break; }}- } else if (arg == "--embedding" || arg == "--embeddings") {+ }+ else if (arg == "--embedding")+ {params.embedding = true;- } else if (arg == "-cb" || arg == "--cont-batching") {+ }+ else if (arg == "-cb" || arg == "--cont-batching")+ {params.cont_batching = true;- } else if (arg == "-np" || arg == "--parallel") {- if (++i >= argc) {+ }+ else if (arg == "-np" || arg == "--parallel")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.n_parallel = std::stoi(argv[i]);- } else if (arg == "-n" || arg == "--n-predict") {- if (++i >= argc) {+ } else if (arg == "-n" || arg == "--n-predict")+ {+ if (++i >= argc)+ {invalid_param = true;break;}params.n_predict = std::stoi(argv[i]);- } else if (arg == "-spf" || arg == "--system-prompt-file") {- if (++i >= argc) {+ } else if (arg == "-spf" || arg == "--system-prompt-file")+ {+ if (++i >= argc)+ {invalid_param = true;break;}@@ -2618,39 +2563,67 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,invalid_param = true;break;}- std::string system_prompt;+ std::string systm_content;std::copy(std::istreambuf_iterator<char>(file),std::istreambuf_iterator<char>(),- std::back_inserter(system_prompt)+ std::back_inserter(systm_content));- sparams.system_prompt = system_prompt;- } else if (arg == "-ctk" || arg == "--cache-type-k") {+ llama.process_system_prompt_data(json::parse(systm_content));+ }+ else if (arg == "-ctk" || arg == "--cache-type-k") {params.cache_type_k = argv[++i];- } else if (arg == "-ctv" || arg == "--cache-type-v") {+ }+ else if (arg == "-ctv" || arg == "--cache-type-v") {params.cache_type_v = argv[++i];- } else if (arg == "--log-format") {- if (++i >= argc) {+ }+ else if(arg == "--mmproj")+ {+ if (++i >= argc)+ {+ invalid_param = true;+ break;+ }+ params.mmproj = argv[i];+ }+ else if (arg == "--log-format")+ {+ if (++i >= argc)+ {invalid_param = true;break;}- if (std::strcmp(argv[i], "json") == 0) {+ if (std::strcmp(argv[i], "json") == 0)+ {server_log_json = true;- } else if (std::strcmp(argv[i], "text") == 0) {+ }+ else if (std::strcmp(argv[i], "text") == 0)+ {server_log_json = false;- } else {+ }+ else+ {invalid_param = true;break;}- } else if (arg == "--log-disable") {+ }+ else if (arg == "--log-disable")+ {log_set_target(stdout);LOG_INFO("logging to file is disabled.", {});- } else if (arg == "--slots-endpoint-disable") {+ }+ else if (arg == "--slots-endpoint-disable")+ {sparams.slots_endpoint = false;- } else if (arg == "--metrics") {+ }+ else if (arg == "--metrics")+ {sparams.metrics_endpoint = true;- } else if (arg == "--chat-template") {- if (++i >= argc) {+ }+ else if (arg == "--chat-template")+ {+ if (++i >= argc)+ {invalid_param = true;break;}@@ -2661,7 +2634,9 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,break;}sparams.chat_template = argv[i];- } else if (arg == "--override-kv") {+ }+ else if (arg == "--override-kv")+ {if (++i >= argc) {invalid_param = true;break;@@ -2672,7 +2647,6 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,invalid_param = true;break;}-struct llama_model_kv_override kvo;std::strncpy(kvo.key, argv[i], sep - argv[i]);kvo.key[sep - argv[i]] = 0;@@ -2703,28 +2677,71 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,break;}params.kv_overrides.push_back(kvo);- } else {+ }+ else+ {fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());server_print_usage(argv[0], default_params, default_sparams);exit(1);}}+ params.embedding = true; // [jart] #243 always enable embedding mode+ params.n_gpu_layers = llamafile_gpu_layers(params.n_gpu_layers);+if (!params.kv_overrides.empty()) {params.kv_overrides.emplace_back();params.kv_overrides.back().key[0] = 0;}- if (invalid_param) {+ if (invalid_param)+ {fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());server_print_usage(argv[0], default_params, default_sparams);exit(1);}}-static void log_server_request(const httplib::Request & req, const httplib::Response & res) {+/* llama.cpp completion api semantics */+static json format_partial_response(+ llama_server_context &llama, llama_client_slot *slot, const std::string &content, const std::vector<completion_token_output> &probs+) {+ json res = json+ {+ {"content", content },+ {"stop", false},+ {"slot_id", slot->id },+ {"multimodal", llama.multimodal }+ };++ if (slot->sparams.n_probs > 0)+ {+ res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);+ }++ return res;+}++static json format_tokenizer_response(const std::vector<llama_token> &tokens)+{+ return json {+ {"tokens", tokens}+ };+}++static json format_detokenized_response(std::string content)+{+ return json {+ {"content", content}+ };+}+++static void log_server_request(const httplib::Request &req, const httplib::Response &res)+{// skip GH copilot requests when using default port- if (req.path == "/v1/health" || req.path == "/v1/completions") {+ if (req.path == "/v1/health" || req.path == "/v1/completions")+ {return;}@@ -2743,164 +2760,308 @@ static void log_server_request(const httplib::Request & req, const httplib::Resp});}-std::function<void(int)> shutdown_handler;-std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;+struct token_translator+{+ llama_context * ctx;+ std::string operator()(llama_token tok) const { return llama_token_to_piece(ctx, tok); }+ std::string operator()(const completion_token_output &cto) const { return (*this)(cto.tok); }+};-inline void signal_handler(int signal) {- if (is_terminating.test_and_set()) {- // in case it hangs, we can force terminate the server by hitting Ctrl+C twice- // this is for better developer experience, we can remove when the server is stable enough- fprintf(stderr, "Received second interrupt, terminating immediately.\n");- exit(1);+static void append_to_generated_text_from_generated_token_probs(llama_server_context &llama, llama_client_slot *slot)+{+ auto & gtps = slot->generated_token_probs;+ auto translator = token_translator{llama.ctx};+ auto add_strlen = [=](size_t sum, const completion_token_output & cto) { return sum + translator(cto).size(); };+ const size_t len = std::accumulate(gtps.begin(), gtps.end(), size_t(0), add_strlen);+ if (slot->generated_text.capacity() < slot->generated_text.size() + len)+ {+ slot->generated_text.reserve(slot->generated_text.size() + len);+ }+ for (const completion_token_output & cto : gtps)+ {+ slot->generated_text += translator(cto);}-- shutdown_handler(signal);}+std::function<void(int)> shutdown_handler;+inline void signal_handler(int signal) { shutdown_handler(signal); }+static const char *sockaddr2str(const struct sockaddr *sa, char *buf, size_t size) {- if (sa->sa_family == AF_INET) {- return inet_ntop(AF_INET, &(((const struct sockaddr_in *)sa)->sin_addr),- buf, size);- } else if (sa->sa_family == AF_INET6) {- return inet_ntop(AF_INET6, &(((const struct sockaddr_in6 *)sa)->sin6_addr),- buf, size);- } else {- return 0;- }+ if (sa->sa_family == AF_INET) {+ return inet_ntop(AF_INET, &(((const struct sockaddr_in *)sa)->sin_addr),+ buf, size);+ } else if (sa->sa_family == AF_INET6) {+ return inet_ntop(AF_INET6, &(((const struct sockaddr_in6 *)sa)->sin6_addr),+ buf, size);+ } else {+ return 0;+ }}-int server_cli(int argc, char ** argv) {+int server_cli(int argc, char **argv)+{#if SERVER_VERBOSE != 1log_disable();#endif// own arguments required by this example- gpt_params params;+ gpt_params params;server_params sparams;// struct that contains llama context and inference- server_context ctx_server;+ llama_server_context llama;- server_params_parse(argc, argv, sparams, params);+ server_params_parse(argc, argv, sparams, params, llama);- if (!sparams.system_prompt.empty()) {- ctx_server.system_prompt_set(json::parse(sparams.system_prompt));- }-- if (params.model_alias == "unknown") {+ if (params.model_alias == "unknown")+ {params.model_alias = params.model;}llama_backend_init();llama_numa_init(params.numa);- LOG_INFO("build info", {- {"build", LLAMA_BUILD_NUMBER},- {"commit", LLAMA_COMMIT}- });+ LOG_INFO("build info", {{"build", LLAMA_BUILD_NUMBER},+ {"commit", LLAMA_COMMIT}});LOG_INFO("system info", {- {"n_threads", params.n_threads},- {"n_threads_batch", params.n_threads_batch},- {"total_threads", std::thread::hardware_concurrency()},- {"system_info", llama_print_system_info()},- });+ {"n_threads", params.n_threads},+ {"n_threads_batch", params.n_threads_batch},+ {"total_threads", std::thread::hardware_concurrency()},+ {"system_info", llama_print_system_info()},+ });- std::unique_ptr<httplib::Server> svr;-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- if (sparams.ssl_key_file != "" && sparams.ssl_cert_file != "") {- LOG_INFO("Running with SSL", {{"key", sparams.ssl_key_file}, {"cert", sparams.ssl_cert_file}});- svr.reset(- new httplib::SSLServer(sparams.ssl_cert_file.c_str(), sparams.ssl_key_file.c_str())- );- } else {- LOG_INFO("Running without SSL", {});- svr.reset(new httplib::Server());- }-#else- svr.reset(new httplib::Server());-#endif+ httplib::Server svr;std::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};- svr->set_default_headers({{"Server", "llama.cpp"}});+ svr.set_default_headers({{"Server", "llama.cpp"}});// CORS preflight- svr->Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ svr.Options(R"(.*)", [](const httplib::Request &req, httplib::Response &res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));res.set_header("Access-Control-Allow-Credentials", "true");- res.set_header("Access-Control-Allow-Methods", "POST");- res.set_header("Access-Control-Allow-Headers", "*");- return res.set_content("", "application/json; charset=utf-8");+ res.set_header("Access-Control-Allow-Methods", "POST");+ res.set_header("Access-Control-Allow-Headers", "*");});- svr->set_logger(log_server_request);+ svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) {+ server_state current_state = state.load();+ switch(current_state) {+ case SERVER_STATE_READY: {+ // request slots data using task queue+ task_server task;+ task.id = llama.queue_tasks.get_new_id();+ task.type = TASK_TYPE_METRICS;+ task.target_id = -1;++ llama.queue_results.add_waiting_task_id(task.id);+ llama.queue_tasks.post(task);- auto res_error = [](httplib::Response & res, json error_data) {- json final_response {{"error", error_data}};- res.set_content(final_response.dump(), "application/json; charset=utf-8");- res.status = json_value(error_data, "code", 500);- };+ // get the result+ task_result result = llama.queue_results.recv(task.id);+ llama.queue_results.remove_waiting_task_id(task.id);- svr->set_exception_handler([&res_error](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) {- std::string message;- try {- std::rethrow_exception(std::move(ep));- } catch (std::exception & e) {- message = e.what();- } catch (...) {- message = "Unknown Exception";- }+ int n_idle_slots = result.result_json["idle"];+ int n_processing_slots = result.result_json["processing"];- json formatted_error = format_error_response(message, ERROR_TYPE_SERVER);- LOG_VERBOSE("Got exception", formatted_error);- res_error(res, formatted_error);- });+ json health = {+ {"status", "ok"},+ {"slots_idle", n_idle_slots},+ {"slots_processing", n_processing_slots}};+ res.status = 200; // HTTP OK+ if (sparams.slots_endpoint && req.has_param("include_slots")) {+ health["slots"] = result.result_json["slots"];+ }- svr->set_error_handler([&res_error](const httplib::Request &, httplib::Response & res) {- if (res.status == 404) {- res_error(res, format_error_response("File Not Found", ERROR_TYPE_NOT_FOUND));+ if (n_idle_slots == 0) {+ health["status"] = "no slot available";+ if (req.has_param("fail_on_no_slot")) {+ res.status = 503; // HTTP Service Unavailable+ }+ }+ res.set_content(health.dump(), "application/json");+ break;+ }+ case SERVER_STATE_LOADING_MODEL:+ res.set_content(R"({"status": "loading model"})", "application/json");+ res.status = 503; // HTTP Service Unavailable+ break;+ case SERVER_STATE_ERROR:+ res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json");+ res.status = 500; // HTTP Internal Server Error+ break;}- // for other error codes, we skip processing here because it's already done by res_error()});- // set timeouts and change hostname and port- svr->set_read_timeout (sparams.read_timeout);- svr->set_write_timeout(sparams.write_timeout);+ if (sparams.slots_endpoint) {+ svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) {+ // request slots data using task queue+ task_server task;+ task.id = llama.queue_tasks.get_new_id();+ task.type = TASK_TYPE_METRICS;+ task.target_id = -1;- std::unordered_map<std::string, std::string> log_data;+ llama.queue_results.add_waiting_task_id(task.id);+ llama.queue_tasks.post(task);- log_data["hostname"] = sparams.hostname;- log_data["port"] = std::to_string(sparams.port);+ // get the result+ task_result result = llama.queue_results.recv(task.id);+ llama.queue_results.remove_waiting_task_id(task.id);- if (sparams.api_keys.size() == 1) {- auto key = sparams.api_keys[0];- log_data["api_key"] = "api_key: ****" + key.substr(std::max((int)(key.length() - 4), 0));- } else if (sparams.api_keys.size() > 1) {- log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";+ res.set_content(result.result_json["slots"].dump(), "application/json");+ res.status = 200; // HTTP OK+ });}- // load the model- if (!ctx_server.load_model(params)) {- state.store(SERVER_STATE_ERROR);- return 1;- } else {- ctx_server.init();- state.store(SERVER_STATE_READY);- }+ if (sparams.metrics_endpoint) {+ svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) {+ // request slots data using task queue+ task_server task;+ task.id = llama.queue_tasks.get_new_id();+ task.type = TASK_TYPE_METRICS;+ task.target_id = -1;- LOG_INFO("model loaded", {});+ llama.queue_results.add_waiting_task_id(task.id);+ llama.queue_tasks.post(task);- for (int i = 0;; ++i) {- if (svr->bind_to_port(sparams.hostname, sparams.port)) {- break;- } else if (i < 10) {- ++sparams.port;- } else {- fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);- return 1;+ // get the result+ task_result result = llama.queue_results.recv(task.id);+ llama.queue_results.remove_waiting_task_id(task.id);++ json data = result.result_json;++ uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"];+ uint64_t t_prompt_processing = data["t_prompt_processing"];++ uint64_t n_tokens_predicted = data["n_tokens_predicted"];+ uint64_t t_tokens_generation = data["t_tokens_generation"];++ int32_t kv_cache_used_cells = data["kv_cache_used_cells"];++ // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names+ json all_metrics_def = json {+ {"counter", {{+ {"name", "prompt_tokens_total"},+ {"help", "Number of prompt tokens processed."},+ {"value", data["n_prompt_tokens_processed_total"]}+ }, {+ {"name", "tokens_predicted_total"},+ {"help", "Number of generation tokens processed."},+ {"value", data["n_tokens_predicted_total"]}+ }}},+ {"gauge", {{+ {"name", "prompt_tokens_seconds"},+ {"help", "Average prompt throughput in tokens/s."},+ {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0}+ },{+ {"name", "predicted_tokens_seconds"},+ {"help", "Average generation throughput in tokens/s."},+ {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0}+ },{+ {"name", "kv_cache_usage_ratio"},+ {"help", "KV-cache usage. 1 means 100 percent usage."},+ {"value", 1. * kv_cache_used_cells / params.n_ctx}+ },{+ {"name", "kv_cache_tokens"},+ {"help", "KV-cache tokens."},+ {"value", data["kv_cache_tokens_count"]}+ },{+ {"name", "requests_processing"},+ {"help", "Number of request processing."},+ {"value", data["processing"]}+ },{+ {"name", "requests_deferred"},+ {"help", "Number of request deferred."},+ {"value", data["deferred"]}+ }}}+ };++ std::stringstream prometheus;+ for (const auto& el : all_metrics_def.items()) {+ const auto& type = el.key();+ const auto& metrics_def = el.value();+ for (const auto& metric_def : metrics_def) {+ std::string name = metric_def["name"];+ std::string help = metric_def["help"];+ prometheus << "# HELP llamacpp:" << name << " " << help << "\n"+ << "# TYPE llamacpp:" << name << " " << type << "\n"+ << "llamacpp:" << name << " " << metric_def["value"] << "\n";+ }+ }++ res.set_content(prometheus.str(), "text/plain; version=0.0.4");+ res.status = 200; // HTTP OK+ });+ }++ svr.set_logger(log_server_request);++ svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep)+ {+ const char fmt[] = "500 Internal Server Error\n%s";+ char buf[BUFSIZ];+ try+ {+ std::rethrow_exception(std::move(ep));+ }+ catch (std::exception &e)+ {+ snprintf(buf, sizeof(buf), fmt, e.what());+ }+ catch (...)+ {+ snprintf(buf, sizeof(buf), fmt, "Unknown Exception");+ }+ res.set_content(buf, "text/plain; charset=utf-8");+ res.status = 500;+ });++ svr.set_error_handler([](const httplib::Request &, httplib::Response &res)+ {+ if (res.status == 401)+ {+ res.set_content("Unauthorized", "text/plain; charset=utf-8");+ }+ if (res.status == 400)+ {+ res.set_content("Invalid request", "text/plain; charset=utf-8");+ }+ else if (res.status == 404)+ {+ res.set_content("File Not Found", "text/plain; charset=utf-8");+ res.status = 404;+ }+ });++ // load the model+ if (!llama.load_model(params))+ {+ state.store(SERVER_STATE_ERROR);+ return 1;+ } else {+ llama.initialize();+ state.store(SERVER_STATE_READY);+ LOG_INFO("model loaded", {});+ }++ // set timeouts and change hostname and port+ svr.set_read_timeout (sparams.read_timeout);+ svr.set_write_timeout(sparams.write_timeout);++ for (int i = 0;; ++i) {+ if (svr.bind_to_port(sparams.hostname, sparams.port)) {+ break;+ } else if (i < 10) {+ ++sparams.port;+ } else {+ fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);+ return 1;}}+ // Set the base directory for serving static files+ svr.set_base_dir(sparams.public_path);+// to make it ctrl+clickable:const char *connect_host;if (sparams.hostname != "0.0.0.0") {@@ -2929,66 +3090,107 @@ int server_cli(int argc, char ** argv) {}}- const auto model_meta = ctx_server.model_meta();+ std::unordered_map<std::string, std::string> log_data;+ log_data["hostname"] = sparams.hostname;+ log_data["port"] = std::to_string(sparams.port);- // if a custom chat template is not supplied, we will use the one that comes with the model (if any)- if (sparams.chat_template.empty()) {- if (!ctx_server.validate_model_chat_template()) {- LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {});- sparams.chat_template = "chatml";+ if (sparams.api_keys.size() == 1) {+ log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4);+ } else if (sparams.api_keys.size() > 1) {+ log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";+ }++ if (!FLAG_unsecure) {+ if (IsXnu()) {+ // Cosmopolitan libc explicitly does not support cosmo_dlopen on x64+ // macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt+ // to enable sandboxing and ignore the failure on x64 macOS.+ // (Alternatively, the sandbox-exec(1) command could be used to+ // launch llamafile with a sandbox profile.) A goal of llamafile is+ // to work easily out-of-the-box and it already operates without a+ // sandbox on macOS so don't require --unsecure on x64 macOS to+ // workaround the failure. If support is added later, it will be+ // used without further changes. On Apple Silicon, if sandbox+ // initialization fails, exit and recommend using --unsecure to+ // disable sandboxing.+ std::string error_msg;+ int rv = mac_sandbox_init(error_msg);+ if (rv != 0) {+ fprintf(stderr, "Sandbox initialization failed:\n"+ "rv: %d, error: %s.\n"+ "Disable sandboxing with the --unsecure option.\n",+ rv, error_msg.c_str());+ if (IsXnuSilicon()) {+ exit(1);+ } else {+ LOG_WARNING("Sandbox initialization failed. Not supported "+ "on x64 macOS. Continuing.", {});+ }+ }+ } else if (!llamafile_has_gpu()) {+ printf("In the sandboxing block!\n");+ // Enables pledge() security on Linux and OpenBSD.+ // - We do this *after* binding the server socket.+ // - We do this *after* opening the log file for writing.+ // - We do this *after* opening a tab in the user browser.+ // - We do this *before* loading any weights or graphdefs.+ // In effect, what this does is:+ // - Filesystem access is disabled entirely (except ZipOS).+ // - On Linux, network access is restricted to accept() only.+ // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.+ char promises[32];+ if (IsOpenbsd()) {+ strlcpy(promises, "stdio inet", sizeof(promises));+ } else {+ strlcpy(promises, "stdio anet", sizeof(promises));+ }+ if (!startswith(sparams.public_path.c_str(), "/zip/")) {+ strlcat(promises, " rpath", sizeof(promises));+ }+ __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;+ if (pledge(0, 0)) {+ LOG_TEE("warning: this OS doesn't support pledge() security\n");+ } else if (pledge(promises, 0)) {+ perror("pledge");+ exit(1);+ }}}- // print sample chat example to make it clear which template is used- {- json chat;- chat.push_back({{"role", "system"}, {"content", "You are a helpful assistant"}});- chat.push_back({{"role", "user"}, {"content", "Hello"}});- chat.push_back({{"role", "assistant"}, {"content", "Hi there"}});- chat.push_back({{"role", "user"}, {"content", "How are you?"}});+ LOG_INFO("HTTP server listening", log_data);+ // run the HTTP server in a thread - see comment below+ std::thread t([&]()+ {+ if (!svr.listen_after_bind())+ {+ state.store(SERVER_STATE_ERROR);+ return 1;+ }- const std::string chat_example = format_chat(ctx_server.model, sparams.chat_template, chat);+ return 0;+ });- LOG_INFO("chat template", {- {"chat_example", chat_example},- {"built_in", sparams.chat_template.empty()},- });+ // launch browser tab+ if (!sparams.nobrowser) {+ char url[128];+ snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);+ llamafile_launch_browser(url);}- //- // Middlewares- //-- auto middleware_validate_api_key = [&sparams, &res_error](const httplib::Request & req, httplib::Response & res) {- // TODO: should we apply API key to all endpoints, including "/health" and "/models"?- static const std::set<std::string> protected_endpoints = {- "/props",- "/completion",- "/completions",- "/v1/completions",- "/chat/completions",- "/v1/chat/completions",- "/infill",- "/tokenize",- "/detokenize",- "/embedding",- "/embeddings",- "/v1/embeddings",- };+ if (sparams.chat_template.empty()) { // custom chat template is not supplied+ // check if the template comes with the model is supported by us+ llama.validate_model_chat_template(sparams);+ }+ // Middleware for API key validation+ auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool {// If API key is not set, skip validationif (sparams.api_keys.empty()) {return true;}- // If path is not in protected_endpoints list, skip validation- if (protected_endpoints.find(req.path) == protected_endpoints.end()) {- return true;- }-// Check for API key in the headerauto auth_header = req.get_header_value("Authorization");-std::string prefix = "Bearer ";if (auth_header.substr(0, prefix.size()) == prefix) {std::string received_api_key = auth_header.substr(prefix.size());@@ -2998,663 +3200,427 @@ int server_cli(int argc, char ** argv) {}// API key is invalid or not provided- // TODO: make another middleware for CORS related logic- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- res_error(res, format_error_response("Invalid API Key", ERROR_TYPE_AUTHENTICATION));+ res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8");+ res.status = 401; // UnauthorizedLOG_WARNING("Unauthorized: Invalid API Key", {});return false;};- // register server middlewares- svr->set_pre_routing_handler([&middleware_validate_api_key](const httplib::Request & req, httplib::Response & res) {- if (!middleware_validate_api_key(req, res)) {- return httplib::Server::HandlerResponse::Handled;- }- return httplib::Server::HandlerResponse::Unhandled;- });-- //- // Route handlers (or controllers)- //-- const auto handle_health = [&](const httplib::Request & req, httplib::Response & res) {- server_state current_state = state.load();- switch (current_state) {- case SERVER_STATE_READY:- {- // request slots data using task queue- server_task task;- task.id = ctx_server.queue_tasks.get_new_id();- task.type = SERVER_TASK_TYPE_METRICS;- task.id_target = -1;-- ctx_server.queue_results.add_waiting_task_id(task.id);- ctx_server.queue_tasks.post(task);-- // get the result- server_task_result result = ctx_server.queue_results.recv(task.id);- ctx_server.queue_results.remove_waiting_task_id(task.id);-- const int n_idle_slots = result.data["idle"];- const int n_processing_slots = result.data["processing"];-- json health = {- {"status", "ok"},- {"slots_idle", n_idle_slots},- {"slots_processing", n_processing_slots}- };+ svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ json data = {+ { "user_name", llama.name_user.c_str() },+ { "assistant_name", llama.name_assistant.c_str() },+ { "default_generation_settings", llama.default_generation_settings_for_props },+ { "total_slots", llama.params.n_parallel }+ };+ res.set_content(data.dump(), "application/json; charset=utf-8");+ });- res.status = 200; // HTTP OK- if (sparams.slots_endpoint && req.has_param("include_slots")) {- health["slots"] = result.data["slots"];+ svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ if (!validate_api_key(req, res)) {+ return;+ }+ json data = json::parse(req.body);+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, data, false, false, -1);+ if (!json_value(data, "stream", false)) {+ std::string completion_text;+ task_result result = llama.queue_results.recv(task_id);+ if (!result.error && result.stop) {+ res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");}-- if (n_idle_slots == 0) {- health["status"] = "no slot available";- if (req.has_param("fail_on_no_slot")) {- res.status = 503; // HTTP Service Unavailable- }+ else+ {+ res.status = 404;+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");}-- res.set_content(health.dump(), "application/json");- break;- }- case SERVER_STATE_LOADING_MODEL:- {- res_error(res, format_error_response("Loading model", ERROR_TYPE_UNAVAILABLE));- } break;- case SERVER_STATE_ERROR:- {- res_error(res, format_error_response("Model failed to load", ERROR_TYPE_SERVER));- } break;- }- };-- const auto handle_slots = [&](const httplib::Request &, httplib::Response & res) {- if (!sparams.slots_endpoint) {- res_error(res, format_error_response("This server does not support slots endpoint.", ERROR_TYPE_NOT_SUPPORTED));- return;- }-- // request slots data using task queue- server_task task;- task.id = ctx_server.queue_tasks.get_new_id();- task.id_multi = -1;- task.id_target = -1;- task.type = SERVER_TASK_TYPE_METRICS;-- ctx_server.queue_results.add_waiting_task_id(task.id);- ctx_server.queue_tasks.post(task);-- // get the result- server_task_result result = ctx_server.queue_results.recv(task.id);- ctx_server.queue_results.remove_waiting_task_id(task.id);-- res.set_content(result.data["slots"].dump(), "application/json");- res.status = 200; // HTTP OK- };-- const auto handle_metrics = [&](const httplib::Request &, httplib::Response & res) {- if (!sparams.metrics_endpoint) {- res_error(res, format_error_response("This server does not support metrics endpoint.", ERROR_TYPE_NOT_SUPPORTED));- return;- }-- // request slots data using task queue- server_task task;- task.id = ctx_server.queue_tasks.get_new_id();- task.id_multi = -1;- task.id_target = -1;- task.type = SERVER_TASK_TYPE_METRICS;- task.data.push_back({{"reset_bucket", true}});-- ctx_server.queue_results.add_waiting_task_id(task.id);- ctx_server.queue_tasks.post(task);-- // get the result- server_task_result result = ctx_server.queue_results.recv(task.id);- ctx_server.queue_results.remove_waiting_task_id(task.id);-- json data = result.data;-- const uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"];- const uint64_t t_prompt_processing = data["t_prompt_processing"];-- const uint64_t n_tokens_predicted = data["n_tokens_predicted"];- const uint64_t t_tokens_generation = data["t_tokens_generation"];-- const int32_t kv_cache_used_cells = data["kv_cache_used_cells"];-- // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names- json all_metrics_def = json {- {"counter", {{- {"name", "prompt_tokens_total"},- {"help", "Number of prompt tokens processed."},- {"value", (uint64_t) data["n_prompt_tokens_processed_total"]}- }, {- {"name", "prompt_seconds_total"},- {"help", "Prompt process time"},- {"value", (uint64_t) data["t_prompt_processing_total"] / 1.e3}- }, {- {"name", "tokens_predicted_total"},- {"help", "Number of generation tokens processed."},- {"value", (uint64_t) data["n_tokens_predicted_total"]}- }, {- {"name", "tokens_predicted_seconds_total"},- {"help", "Predict process time"},- {"value", (uint64_t) data["t_tokens_generation_total"] / 1.e3}- }}},- {"gauge", {{- {"name", "prompt_tokens_seconds"},- {"help", "Average prompt throughput in tokens/s."},- {"value", n_prompt_tokens_processed ? 1.e3 / t_prompt_processing * n_prompt_tokens_processed : 0.}- },{- {"name", "predicted_tokens_seconds"},- {"help", "Average generation throughput in tokens/s."},- {"value", n_tokens_predicted ? 1.e3 / t_tokens_generation * n_tokens_predicted : 0.}- },{- {"name", "kv_cache_usage_ratio"},- {"help", "KV-cache usage. 1 means 100 percent usage."},- {"value", 1. * kv_cache_used_cells / params.n_ctx}- },{- {"name", "kv_cache_tokens"},- {"help", "KV-cache tokens."},- {"value", (uint64_t) data["kv_cache_tokens_count"]}- },{- {"name", "requests_processing"},- {"help", "Number of request processing."},- {"value", (uint64_t) data["processing"]}- },{- {"name", "requests_deferred"},- {"help", "Number of request deferred."},- {"value", (uint64_t) data["deferred"]}- }}}- };-- std::stringstream prometheus;-- for (const auto & el : all_metrics_def.items()) {- const auto & type = el.key();- const auto & metrics_def = el.value();-- for (const auto & metric_def : metrics_def) {- const std::string name = metric_def["name"];- const std::string help = metric_def["help"];-- auto value = json_value(metric_def, "value", 0.);- prometheus << "# HELP llamacpp:" << name << " " << help << "\n"- << "# TYPE llamacpp:" << name << " " << type << "\n"- << "llamacpp:" << name << " " << value << "\n";- }- }-- const int64_t t_start = data["t_start"];- res.set_header("Process-Start-Time-Unix", std::to_string(t_start));-- res.set_content(prometheus.str(), "text/plain; version=0.0.4");- res.status = 200; // HTTP OK- };-- const auto handle_props = [&ctx_server](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- json data = {- { "user_name", ctx_server.name_user.c_str() },- { "assistant_name", ctx_server.name_assistant.c_str() },- { "default_generation_settings", ctx_server.default_generation_settings_for_props },- { "total_slots", ctx_server.params.n_parallel }- };-- res.set_content(data.dump(), "application/json; charset=utf-8");- };-- const auto handle_completions = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));-- json data = json::parse(req.body);-- const int id_task = ctx_server.queue_tasks.get_new_id();-- ctx_server.queue_results.add_waiting_task_id(id_task);- ctx_server.request_completion(id_task, -1, data, false, false);-- if (!json_value(data, "stream", false)) {- server_task_result result = ctx_server.queue_results.recv(id_task);- if (!result.error && result.stop) {- res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");- } else {- res_error(res, result.data);- }-- ctx_server.queue_results.remove_waiting_task_id(id_task);- } else {- const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) {- while (true) {- server_task_result result = ctx_server.queue_results.recv(id_task);- if (!result.error) {- const std::string str =- "data: " +- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";-- LOG_VERBOSE("data stream", {- { "to_send", str }- });-- if (!sink.write(str.c_str(), str.size())) {- ctx_server.queue_results.remove_waiting_task_id(id_task);- return false;- }-- if (result.stop) {- break;+ llama.queue_results.remove_waiting_task_id(task_id);+ } else {+ const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink)+ {+ while (true)+ {+ task_result result = llama.queue_results.recv(task_id);+ if (!result.error) {+ const std::string str =+ "data: " ++ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";+ LOG_VERBOSE("data stream", {+ { "to_send", str }+ });+ if (!sink.write(str.c_str(), str.size()))+ {+ llama.queue_results.remove_waiting_task_id(task_id);+ return false;+ }+ if (result.stop) {+ break;+ }+ } else {+ const std::string str =+ "error: " ++ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";+ LOG_VERBOSE("data stream", {+ { "to_send", str }+ });+ if (!sink.write(str.c_str(), str.size()))+ {+ llama.queue_results.remove_waiting_task_id(task_id);+ return false;+ }+ break;+ }}- } else {- const std::string str =- "error: " +- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";- LOG_VERBOSE("data stream", {- { "to_send", str }- });+ llama.queue_results.remove_waiting_task_id(task_id);+ sink.done();+ return true;+ };- if (!sink.write(str.c_str(), str.size())) {- ctx_server.queue_results.remove_waiting_task_id(id_task);- return false;- }+ auto on_complete = [task_id, &llama] (bool)+ {+ // cancel+ llama.request_cancel(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);+ };- break;- }+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);}+ });- ctx_server.queue_results.remove_waiting_task_id(id_task);- sink.done();-- return true;- };-- auto on_complete = [id_task, &ctx_server] (bool) {- // cancel- ctx_server.request_cancel(id_task);- ctx_server.queue_results.remove_waiting_task_id(id_task);- };-- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);- }- };-- const auto handle_models = [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));-- json models = {- {"object", "list"},- {"data", {- {- {"id", params.model_alias},- {"object", "model"},- {"created", std::time(0)},- {"owned_by", "llamacpp"},- {"meta", model_meta}- },- }}- };+ svr.Get("/v1/models", [¶ms](const httplib::Request& req, httplib::Response& res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ std::time_t t = std::time(0);++ json models = {+ {"object", "list"},+ {"data", {+ {+ {"id", params.model_alias},+ {"object", "model"},+ {"created", t},+ {"owned_by", "llamacpp"}+ },+ }}+ };- res.set_content(models.dump(), "application/json; charset=utf-8");- };+ res.set_content(models.dump(), "application/json; charset=utf-8");+ });- const auto handle_chat_completions = [&ctx_server, &sparams, &res_error](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), sparams.chat_template);- const int id_task = ctx_server.queue_tasks.get_new_id();+ // TODO: add mount point without "/v1" prefix -- how?+ svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ if (!validate_api_key(req, res)) {+ return;+ }+ json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template);- ctx_server.queue_results.add_waiting_task_id(id_task);- ctx_server.request_completion(id_task, -1, data, false, false);+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, data, false, false, -1);- const auto completion_id = gen_chatcmplid();- if (!json_value(data, "stream", false)) {- server_task_result result = ctx_server.queue_results.recv(id_task);+ if (!json_value(data, "stream", false)) {+ std::string completion_text;+ task_result result = llama.queue_results.recv(task_id);- if (!result.error && result.stop) {- json result_oai = format_final_response_oaicompat(data, result.data, completion_id);+ if (!result.error && result.stop) {+ json oaicompat_result = format_final_response_oaicompat(data, result);- res.set_content(result_oai.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");- } else {- res_error(res, result.data);- }- ctx_server.queue_results.remove_waiting_task_id(id_task);- } else {- const auto chunked_content_provider = [id_task, &ctx_server, completion_id](size_t, httplib::DataSink & sink) {- while (true) {- server_task_result result = ctx_server.queue_results.recv(id_task);- if (!result.error) {- std::vector<json> result_array = format_partial_response_oaicompat(result.data, completion_id);-- for (auto it = result_array.begin(); it != result_array.end(); ++it) {- if (!it->empty()) {+ res.set_content(oaicompat_result.dump(-1, ' ', false,+ json::error_handler_t::replace),+ "application/json; charset=utf-8");+ } else {+ res.status = 500;+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");+ }+ llama.queue_results.remove_waiting_task_id(task_id);+ } else {+ const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) {+ while (true) {+ task_result llama_result = llama.queue_results.recv(task_id);+ if (!llama_result.error) {+ std::vector<json> result_array = format_partial_response_oaicompat( llama_result);++ for (auto it = result_array.begin(); it != result_array.end(); ++it)+ {+ if (!it->empty()) {+ const std::string str =+ "data: " ++ it->dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";+ LOG_VERBOSE("data stream", {{"to_send", str}});+ if (!sink.write(str.c_str(), str.size())) {+ llama.queue_results.remove_waiting_task_id(task_id);+ return false;+ }+ }+ }+ if (llama_result.stop) {+ break;+ }+ } else {const std::string str =- "data: " +- it->dump(-1, ' ', false, json::error_handler_t::replace) ++ "error: " ++ llama_result.result_json.dump(-1, ' ', false,+ json::error_handler_t::replace) +"\n\n";LOG_VERBOSE("data stream", {{"to_send", str}});if (!sink.write(str.c_str(), str.size())) {- ctx_server.queue_results.remove_waiting_task_id(id_task);+ llama.queue_results.remove_waiting_task_id(task_id);return false;}+ break;}}- if (result.stop) {- break;- }- } else {- const std::string str =- "error: " +- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";- LOG_VERBOSE("data stream", {{"to_send", str}});- if (!sink.write(str.c_str(), str.size())) {- ctx_server.queue_results.remove_waiting_task_id(id_task);- return false;- }- break;- }- }- sink.done();- ctx_server.queue_results.remove_waiting_task_id(id_task);- return true;- };-- auto on_complete = [id_task, &ctx_server](bool) {- // cancel request- ctx_server.request_cancel(id_task);- ctx_server.queue_results.remove_waiting_task_id(id_task);- };-- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);- }- };-- const auto handle_infill = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));-- json data = json::parse(req.body);-- const int id_task = ctx_server.queue_tasks.get_new_id();-- ctx_server.queue_results.add_waiting_task_id(id_task);- ctx_server.request_completion(id_task, -1, data, true, false);-- if (!json_value(data, "stream", false)) {- server_task_result result = ctx_server.queue_results.recv(id_task);- if (!result.error && result.stop) {- res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");- } else {- res_error(res, result.data);- }-- ctx_server.queue_results.remove_waiting_task_id(id_task);- } else {- const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) {- while (true) {- server_task_result result = ctx_server.queue_results.recv(id_task);- if (!result.error) {- const std::string str =- "data: " +- result.data.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";-- LOG_VERBOSE("data stream", {- { "to_send", str }- });+ sink.done();+ llama.queue_results.remove_waiting_task_id(task_id);+ return true;+ };- if (!sink.write(str.c_str(), str.size())) {- ctx_server.queue_results.remove_waiting_task_id(id_task);- return false;- }+ auto on_complete = [task_id, &llama](bool) {+ // cancel request+ llama.request_cancel(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);+ };- if (result.stop) {- break;- }- } else {- break;- }+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);}+ });- ctx_server.queue_results.remove_waiting_task_id(id_task);- sink.done();-- return true;- };-- auto on_complete = [id_task, &ctx_server] (bool) {- ctx_server.request_cancel(id_task);- };-- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);- }- };-- const auto handle_tokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- const json body = json::parse(req.body);+ svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ if (!validate_api_key(req, res)) {+ return;+ }+ json data = json::parse(req.body);+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, data, true, false, -1);+ if (!json_value(data, "stream", false)) {+ std::string completion_text;+ task_result result = llama.queue_results.recv(task_id);+ if (!result.error && result.stop)+ {+ res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");+ }+ else+ {+ res.status = 404;+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");+ }+ llama.queue_results.remove_waiting_task_id(task_id);+ } else {+ const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink) {+ while (true)+ {+ task_result result = llama.queue_results.recv(task_id);+ if (!result.error) {+ const std::string str =+ "data: " ++ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";+ LOG_VERBOSE("data stream", {+ { "to_send", str }+ });+ if (!sink.write(str.c_str(), str.size()))+ {+ llama.queue_results.remove_waiting_task_id(task_id);+ return false;+ }+ if (result.stop)+ {+ break;+ }+ }+ else+ {+ break;+ }+ }- std::vector<llama_token> tokens;- if (body.count("content") != 0) {- tokens = ctx_server.tokenize(body["content"], false);- }- const json data = format_tokenizer_response(tokens);- return res.set_content(data.dump(), "application/json; charset=utf-8");- };+ llama.queue_results.remove_waiting_task_id(task_id);+ sink.done();+ return true;+ };- const auto handle_detokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- const json body = json::parse(req.body);+ auto on_complete = [task_id, &llama] (bool)+ {+ // cancel+ llama.request_cancel(task_id);+ };- std::string content;- if (body.count("tokens") != 0) {- const std::vector<llama_token> tokens = body["tokens"];- content = tokens_to_str(ctx_server.ctx, tokens.cbegin(), tokens.cend());- }+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);+ }+ });- const json data = format_detokenized_response(content);- return res.set_content(data.dump(), "application/json; charset=utf-8");- };+ svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res)+ { return res.set_content("", "application/json; charset=utf-8"); });- const auto handle_embeddings = [¶ms, &ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- if (!params.embedding) {- res.status = 501;- res.set_content("This server does not support embeddings. Start it with `--embeddings`", "text/plain; charset=utf-8");- return;- }+ svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ const json body = json::parse(req.body);+ std::vector<llama_token> tokens;+ if (body.count("content") != 0)+ {+ tokens = llama.tokenize(body["content"], false);+ }+ const json data = format_tokenizer_response(tokens);+ return res.set_content(data.dump(), "application/json; charset=utf-8");+ });- const json body = json::parse(req.body);- bool is_openai = false;+ svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ const json body = json::parse(req.body);+ std::string content;+ if (body.count("tokens") != 0)+ {+ const std::vector<llama_token> tokens = body["tokens"];+ content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());+ }- // an input prompt can be a string or a list of tokens (integer)- json prompt;- if (body.count("input") != 0) {- is_openai = true;- prompt = body["input"];- } else if (body.count("content") != 0) {- // with "content", we only support single prompt- prompt = std::vector<std::string>{body["content"]};- } else {- res_error(res, format_error_response("\"input\" or \"content\" must be provided", ERROR_TYPE_INVALID_REQUEST));- return;- }+ const json data = format_detokenized_response(content);+ return res.set_content(data.dump(), "application/json; charset=utf-8");+ });- // create and queue the task- json responses;- {- const int id_task = ctx_server.queue_tasks.get_new_id();- ctx_server.queue_results.add_waiting_task_id(id_task);- ctx_server.request_completion(id_task, -1, {{"prompt", prompt}}, false, true);+ svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ const json body = json::parse(req.body);+ json prompt;+ if (body.count("content") != 0)+ {+ prompt = body["content"];+ }+ else+ {+ prompt = "";+ }- // get the result- server_task_result result = ctx_server.queue_results.recv(id_task);- ctx_server.queue_results.remove_waiting_task_id(id_task);- if (!result.error) {- if (result.data.count("results")) {- // result for multi-task- responses = result.data["results"];- } else {- // result for single task- responses = std::vector<json>{result.data};+ json image_data;+ if (body.count("image_data") != 0) {+ image_data = body["image_data"];+ }+ else+ {+ image_data = "";}- } else {- // error received, ignore everything else- res_error(res, result.data);- return;- }- }- // write JSON response- json root = is_openai- ? format_embeddings_response_oaicompat(body, responses)- : responses[0];- return res.set_content(root.dump(), "application/json; charset=utf-8");- };+ // create and queue the task+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1);- auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) {- return [content, len, mime_type](const httplib::Request &, httplib::Response & res) {- res.set_content(reinterpret_cast<const char*>(content), len, mime_type);- return false;- };- };+ // get the result+ task_result result = llama.queue_results.recv(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);- //- // Router- //+ // send the result+ return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");+ });- // register static assets routes- if (!sparams.public_path.empty()) {- // Set the base directory for serving static files- svr->set_base_dir(sparams.public_path);- }+ svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ const json body = json::parse(req.body);- // register API routes- svr->Get ("/health", handle_health);- svr->Get ("/slots", handle_slots);- svr->Get ("/metrics", handle_metrics);- svr->Get ("/props", handle_props);- svr->Get ("/v1/models", handle_models);- svr->Post("/completion", handle_completions); // legacy- svr->Post("/completions", handle_completions);- svr->Post("/v1/completions", handle_completions);- svr->Post("/chat/completions", handle_chat_completions);- svr->Post("/v1/chat/completions", handle_chat_completions);- svr->Post("/infill", handle_infill);- svr->Post("/embedding", handle_embeddings); // legacy- svr->Post("/embeddings", handle_embeddings);- svr->Post("/v1/embeddings", handle_embeddings);- svr->Post("/tokenize", handle_tokenize);- svr->Post("/detokenize", handle_detokenize);-- //- // Start the server- //- if (sparams.n_threads_http < 1) {- // +2 threads for monitoring endpoints- sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1);- }- log_data["n_threads_http"] = std::to_string(sparams.n_threads_http);- svr->new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); };+ json prompt;+ if (body.count("input") != 0)+ {+ prompt = body["input"];+ // batch+ if(prompt.is_array()) {+ json data = json::array();+ int i = 0;+ for (const json &elem : prompt) {+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, { {"prompt", elem}, { "n_predict", 0} }, false, true, -1);++ // get the result+ task_result result = llama.queue_results.recv(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);++ json embedding = json{+ {"embedding", json_value(result.result_json, "embedding", json::array())},+ {"index", i++},+ {"object", "embedding"}+ };+ data.push_back(embedding);+ }+ json result = format_embeddings_response_oaicompat(body, data);+ return res.set_content(result.dump(), "application/json; charset=utf-8");+ }+ }+ else+ {+ prompt = "";+ }- LOG_INFO("HTTP server listening", log_data);+ // create and queue the task+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}}, false, true, -1);- // launch browser tab- if (!sparams.nobrowser) {- char url[128];- snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);- llamafile_launch_browser(url);- }+ // get the result+ task_result result = llama.queue_results.recv(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);- if (!FLAG_unsecure) {- if (IsXnu()) {- // Cosmopolitan libc explicitly does not support cosmo_dlopen on x64- // macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt- // to enable sandboxing and ignore the failure on x64 macOS.- // (Alternatively, the sandbox-exec(1) command could be used to- // launch llamafile with a sandbox profile.) A goal of llamafile is- // to work easily out-of-the-box and it already operates without a- // sandbox on macOS so don't require --unsecure on x64 macOS to- // workaround the failure. If support is added later, it will be- // used without further changes. On Apple Silicon, if sandbox- // initialization fails, exit and recommend using --unsecure to- // disable sandboxing.- std::string error_msg;- int rv = mac_sandbox_init(error_msg);- if (rv != 0) {- fprintf(stderr, "Sandbox initialization failed:\n"- "rv: %d, error: %s.\n"- "Disable sandboxing with the --unsecure option.\n",- rv, error_msg.c_str());- if (IsXnuSilicon()) {- exit(1);- } else {- LOG_WARNING("Sandbox initialization failed. Not supported "- "on x64 macOS. Continuing.", {});- }- }- } else if (!llamafile_has_gpu()) {- printf("In the sandboxing block!\n");- // Enables pledge() security on Linux and OpenBSD.- // - We do this *after* binding the server socket.- // - We do this *after* opening the log file for writing.- // - We do this *after* opening a tab in the user browser.- // - We do this *before* loading any weights or graphdefs.- // In effect, what this does is:- // - Filesystem access is disabled entirely (except ZipOS).- // - On Linux, network access is restricted to accept() only.- // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.- char promises[32];- if (IsOpenbsd()) {- strlcpy(promises, "stdio inet", sizeof(promises));- } else {- strlcpy(promises, "stdio anet", sizeof(promises));- }- if (!startswith(sparams.public_path.c_str(), "/zip/")) {- strlcat(promises, " rpath", sizeof(promises));- }- __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;- if (pledge(0, 0)) {- LOG_TEE("warning: this OS doesn't support pledge() security\n");- } else if (pledge(promises, 0)) {- perror("pledge");- exit(1);- }- }- }+ json data = json::array({json{+ {"embedding", json_value(result.result_json, "embedding", json::array())},+ {"index", 0},+ {"object", "embedding"}+ }}+ );- // run the HTTP server in a thread - see comment below- std::thread t([&]() {- if (!svr->listen_after_bind()) {- state.store(SERVER_STATE_ERROR);- return 1;- }+ json root = format_embeddings_response_oaicompat(body, data);- return 0;- });+ // send the result+ return res.set_content(root.dump(), "application/json; charset=utf-8");+ });- ctx_server.queue_tasks.on_new_task(std::bind(- &server_context::process_single_task, &ctx_server, std::placeholders::_1));- ctx_server.queue_tasks.on_finish_multitask(std::bind(- &server_context::on_finish_multitask, &ctx_server, std::placeholders::_1));- ctx_server.queue_tasks.on_update_slots(std::bind(- &server_context::update_slots, &ctx_server));- ctx_server.queue_results.on_multitask_update(std::bind(- &server_queue::update_multitask,- &ctx_server.queue_tasks,+ // GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!?+ // "Bus error: 10" - this is on macOS, it does not crash on Linux+ //std::thread t2([&]()+ /*{+ bool running = true;+ while (running)+ {+ running = llama.update_slots();+ }+ }*/+ //);++ llama.queue_tasks.on_new_task(std::bind(+ &llama_server_context::process_single_task, &llama, std::placeholders::_1));+ llama.queue_tasks.on_finish_multitask(std::bind(+ &llama_server_context::on_finish_multitask, &llama, std::placeholders::_1));+ llama.queue_tasks.on_all_tasks_finished(std::bind(+ &llama_server_context::run_on_all_tasks_finished, &llama));+ llama.queue_results.on_multitask_update(std::bind(+ &llama_server_queue::update_multitask,+ &llama.queue_tasks,std::placeholders::_1,std::placeholders::_2,std::placeholders::_3));shutdown_handler = [&](int) {- ctx_server.queue_tasks.terminate();+ llama.queue_tasks.terminate();};#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))@@ -3669,13 +3635,10 @@ int server_cli(int argc, char ** argv) {};SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);#endif-- ctx_server.queue_tasks.start_loop();-- svr->stop();+ llama.queue_tasks.start_loop();+ svr.stop();t.join();llama_backend_free();-return 0;}diff --git a/llama.cpp/server/utils.h b/llama.cpp/server/utils.hindex b84cf70e..ca9b7bdc 100644--- a/llama.cpp/server/utils.h+++ b/llama.cpp/server/utils.h@@ -1,31 +1,20 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#pragma once-#include "llama.cpp/llama.h"-#include "llama.cpp/common.h"-#include "llama.cpp/json.h"+#pragma once#include <string>#include <vector>-#include <sstream>-#include <random>+#include <set>+#include <mutex>+#include <condition_variable>+#include <unordered_map>-#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"+#include "llama.cpp/json.h"+#include "llama.cpp/llava/clip.h"using json = nlohmann::json;-// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/11-enum error_type {- ERROR_TYPE_INVALID_REQUEST,- ERROR_TYPE_AUTHENTICATION,- ERROR_TYPE_SERVER,- ERROR_TYPE_NOT_FOUND,- ERROR_TYPE_PERMISSION,- ERROR_TYPE_UNAVAILABLE, // custom error- ERROR_TYPE_NOT_SUPPORTED, // custom error-};-extern bool server_verbose;extern bool server_log_json;@@ -50,35 +39,125 @@ extern bool server_log_json;#define LOG_WARNING(MSG, ...) server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__)#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__)-template <typename T>-static T json_value(const json &body, const std::string &key, const T &default_value) {- // Fallback null to default value- return body.contains(key) && !body.at(key).is_null()- ? body.value(key, default_value)- : default_value;-}+//+// parallel+//++enum server_state {+ SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet+ SERVER_STATE_READY, // Server is ready and model is loaded+ SERVER_STATE_ERROR // An error occurred, load_model failed+};++enum task_type {+ TASK_TYPE_COMPLETION,+ TASK_TYPE_CANCEL,+ TASK_TYPE_NEXT_RESPONSE,+ TASK_TYPE_METRICS+};++struct task_server {+ int id = -1; // to be filled by llama_server_queue+ int target_id;+ task_type type;+ json data;+ bool infill_mode = false;+ bool embedding_mode = false;+ int multitask_id = -1;+};++struct task_result {+ int id;+ int multitask_id = -1;+ bool stop;+ bool error;+ json result_json;+};++struct task_multi {+ int id;+ std::set<int> subtasks_remaining{};+ std::vector<task_result> results{};+};++// TODO: can become bool if we can't find use of more states+enum slot_state+{+ IDLE,+ PROCESSING,+};++enum slot_command+{+ NONE,+ LOAD_PROMPT,+ RELEASE,+};++struct slot_params+{+ bool stream = true;+ bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt++ uint32_t seed = -1; // RNG seed+ int32_t n_keep = 0; // number of tokens to keep from initial prompt+ int32_t n_predict = -1; // new tokens to predict-static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) {+ std::vector<std::string> antiprompt;++ json input_prefix;+ json input_suffix;+};++struct slot_image+{+ int32_t id;++ bool request_encode_image = false;+ float * image_embedding = nullptr;+ int32_t image_tokens = 0;++ clip_image_u8 * img_data;++ std::string prefix_prompt; // before of this image+};++// completion token output with probabilities+struct completion_token_output+{+ struct token_prob+ {+ llama_token tok;+ float prob;+ };++ std::vector<token_prob> probs;+ llama_token tok;+ std::string text_to_send;+};++static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra)+{std::stringstream ss_tid;ss_tid << std::this_thread::get_id();json log = nlohmann::ordered_json{- {"tid", ss_tid.str()},+ {"tid", ss_tid.str()},{"timestamp", time(nullptr)},};if (server_log_json) {- log.merge_patch( {- {"level", level},- {"function", function},- {"line", line},- {"msg", message},- });-+ log.merge_patch(+ {+ {"level", level},+ {"function", function},+ {"line", line},+ {"msg", message},+ });if (!extra.empty()) {log.merge_patch(extra);}- printf("%s\n", log.dump(-1, ' ', false, json::error_handler_t::replace).c_str());+ std::cout << log.dump(-1, ' ', false, json::error_handler_t::replace) << "\n" << std::flush;} else {char buf[1024];snprintf(buf, 1024, "%4s [%24s] %s", level, function, message);@@ -91,7 +170,8 @@ static inline void server_log(const char *level, const char *function, int line,for (const auto& el : log.items()){const std::string value = el.value().dump(-1, ' ', false, json::error_handler_t::replace);- ss << " " << el.key() << "=" << value;+ snprintf(buf, 1024, " %s=%s", el.key().c_str(), value.c_str());+ ss << buf;}const std::string str = ss.str();@@ -101,25 +181,36 @@ static inline void server_log(const char *level, const char *function, int line,}//-// chat template utils+// server utils//+template <typename T>+static T json_value(const json &body, const std::string &key, const T &default_value)+{+ // Fallback null to default value+ return body.contains(key) && !body.at(key).is_null()+ ? body.value(key, default_value)+ : default_value;+}+// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's validinline bool verify_custom_template(const std::string & tmpl) {llama_chat_message chat[] = {{"user", "test"}};- int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, nullptr, 0);+ std::vector<char> buf(1);+ int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, buf.data(), buf.size());return res >= 0;}// Format given chat. If tmpl is empty, we take the template from model metadata-inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages) {+inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages)+{size_t alloc_size = 0;// vector holding all allocated string to be passed to llama_chat_apply_templatestd::vector<std::string> str(messages.size() * 2);std::vector<llama_chat_message> chat(messages.size());for (size_t i = 0; i < messages.size(); ++i) {- const auto & curr_msg = messages[i];+ auto &curr_msg = messages[i];str[i*2 + 0] = json_value(curr_msg, "role", std::string(""));str[i*2 + 1] = json_value(curr_msg, "content", std::string(""));alloc_size += str[i*2 + 1].length();@@ -139,13 +230,252 @@ inline std::string format_chat(const struct llama_model * model, const std::strires = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size());}- const std::string formatted_chat(buf.data(), res);-+ std::string formatted_chat(buf.data(), res);LOG_VERBOSE("formatted_chat", {{"text", formatted_chat.c_str()}});return formatted_chat;}+//+// work queue utils+//++struct llama_server_queue {+ int id = 0;+ std::mutex mutex_tasks;+ bool running;+ // queues+ std::vector<task_server> queue_tasks;+ std::vector<task_server> queue_tasks_deferred;+ std::vector<task_multi> queue_multitasks;+ std::condition_variable condition_tasks;+ // callback functions+ std::function<void(task_server&)> callback_new_task;+ std::function<void(task_multi&)> callback_finish_multitask;+ std::function<void(void)> callback_all_task_finished;++ // Add a new task to the end of the queue+ int post(task_server task) {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (task.id == -1) {+ task.id = id++;+ LOG_VERBOSE("new task id", {{"new_id", task.id}});+ }+ queue_tasks.push_back(std::move(task));+ condition_tasks.notify_one();+ return task.id;+ }++ // Add a new task, but defer until one slot is available+ void defer(task_server task) {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ queue_tasks_deferred.push_back(std::move(task));+ }++ // Get the next id for creating anew task+ int get_new_id() {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ int new_id = id++;+ LOG_VERBOSE("new task id", {{"new_id", new_id}});+ return new_id;+ }++ // Register function to process a new task+ void on_new_task(std::function<void(task_server&)> callback) {+ callback_new_task = callback;+ }++ // Register function to process a multitask+ void on_finish_multitask(std::function<void(task_multi&)> callback) {+ callback_finish_multitask = callback;+ }++ // Register the function to be called when the batch of tasks is finished+ void on_all_tasks_finished(std::function<void(void)> callback) {+ callback_all_task_finished = callback;+ }++ // Call when the state of one slot is changed+ void notify_slot_changed() {+ // move deferred tasks back to main loop+ std::unique_lock<std::mutex> lock(mutex_tasks);+ for (auto & task : queue_tasks_deferred) {+ queue_tasks.push_back(std::move(task));+ }+ queue_tasks_deferred.clear();+ }++ // end the start_loop routine+ void terminate() {+ {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ running = false;+ }+ condition_tasks.notify_all();+ }++ // Start the main loop.+ void start_loop() {+ running = true;+ while (true) {+ LOG_VERBOSE("new task may arrive", {});+ {+ while (true)+ {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (queue_tasks.empty()) {+ lock.unlock();+ break;+ }+ task_server task = queue_tasks.front();+ queue_tasks.erase(queue_tasks.begin());+ lock.unlock();+ LOG_VERBOSE("callback_new_task", {{"task_id", task.id}});+ callback_new_task(task);+ }+ LOG_VERBOSE("callback_all_task_finished", {});+ // process and update all the multitasks+ auto queue_iterator = queue_multitasks.begin();+ while (queue_iterator != queue_multitasks.end())+ {+ if (queue_iterator->subtasks_remaining.empty())+ {+ // all subtasks done == multitask is done+ task_multi current_multitask = *queue_iterator;+ callback_finish_multitask(current_multitask);+ // remove this multitask+ queue_iterator = queue_multitasks.erase(queue_iterator);+ }+ else+ {+ ++queue_iterator;+ }+ }+ // all tasks in the current loop is finished+ callback_all_task_finished();+ }+ LOG_VERBOSE("wait for new task", {});+ // wait for new task+ {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (queue_tasks.empty()) {+ if (!running) {+ LOG_VERBOSE("ending start_loop", {});+ return;+ }+ condition_tasks.wait(lock, [&]{+ return (!queue_tasks.empty() || !running);+ });+ }+ }+ }+ }++ //+ // functions to manage multitasks+ //++ // add a multitask by specifying the id of all subtask (subtask is a task_server)+ void add_multitask(int multitask_id, std::vector<int>& sub_ids)+ {+ std::lock_guard<std::mutex> lock(mutex_tasks);+ task_multi multi;+ multi.id = multitask_id;+ std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));+ queue_multitasks.push_back(multi);+ }++ // updatethe remaining subtasks, while appending results to multitask+ void update_multitask(int multitask_id, int subtask_id, task_result& result)+ {+ std::lock_guard<std::mutex> lock(mutex_tasks);+ for (auto& multitask : queue_multitasks)+ {+ if (multitask.id == multitask_id)+ {+ multitask.subtasks_remaining.erase(subtask_id);+ multitask.results.push_back(result);+ }+ }+ }+};++struct llama_server_response {+ typedef std::function<void(int, int, task_result&)> callback_multitask_t;+ callback_multitask_t callback_update_multitask;+ // for keeping track of all tasks waiting for the result+ std::set<int> waiting_task_ids;+ // the main result queue+ std::vector<task_result> queue_results;+ std::mutex mutex_results;+ std::condition_variable condition_results;++ void add_waiting_task_id(int task_id) {+ LOG_VERBOSE("waiting for task id", {{"task_id", task_id}});+ std::unique_lock<std::mutex> lock(mutex_results);+ waiting_task_ids.insert(task_id);+ }++ void remove_waiting_task_id(int task_id) {+ LOG_VERBOSE("remove waiting for task id", {{"task_id", task_id}});+ std::unique_lock<std::mutex> lock(mutex_results);+ waiting_task_ids.erase(task_id);+ }++ // This function blocks the thread until there is a response for this task_id+ task_result recv(int task_id) {+ while (true)+ {+ std::unique_lock<std::mutex> lock(mutex_results);+ condition_results.wait(lock, [&]{+ return !queue_results.empty();+ });++ for (int i = 0; i < (int) queue_results.size(); i++)+ {+ if (queue_results[i].id == task_id)+ {+ assert(queue_results[i].multitask_id == -1);+ task_result res = queue_results[i];+ queue_results.erase(queue_results.begin() + i);+ return res;+ }+ }+ }++ // should never reach here+ }++ // Register the function to update multitask+ void on_multitask_update(callback_multitask_t callback) {+ callback_update_multitask = callback;+ }++ // Send a new result to a waiting task_id+ void send(task_result result) {+ std::unique_lock<std::mutex> lock(mutex_results);+ LOG_VERBOSE("send new result", {{"task_id", result.id}});+ for (auto& task_id : waiting_task_ids) {+ // LOG_TEE("waiting task id %i \n", task_id);+ // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result+ if (result.multitask_id == task_id)+ {+ LOG_VERBOSE("callback_update_multitask", {{"task_id", task_id}});+ callback_update_multitask(task_id, result.id, result);+ continue;+ }++ if (result.id == task_id)+ {+ LOG_VERBOSE("queue_results.push_back", {{"task_id", task_id}});+ queue_results.push_back(result);+ condition_results.notify_all();+ return;+ }+ }+ }+};+//// base64 utils (TODO: move to common in the future)//@@ -155,11 +485,13 @@ static const std::string base64_chars ="abcdefghijklmnopqrstuvwxyz""0123456789+/";-static inline bool is_base64(uint8_t c) {+static inline bool is_base64(uint8_t c)+{return (isalnum(c) || (c == '+') || (c == '/'));}-static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string) {+static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string)+{int i = 0;int j = 0;int in_ = 0;@@ -171,10 +503,13 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_strstd::vector<uint8_t> ret;- while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {+ while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))+ {char_array_4[i++] = encoded_string[in_]; in_++;- if (i == 4) {- for (i = 0; i < 4; i++) {+ if (i == 4)+ {+ for (i = 0; i <4; i++)+ {char_array_4[i] = base64_chars.find(char_array_4[i]);}@@ -182,20 +517,23 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_strchar_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];- for (i = 0; (i < 3); i++) {+ for (i = 0; (i < 3); i++)+ {ret.push_back(char_array_3[i]);}-i = 0;}}- if (i) {- for (j = i; j < 4; j++) {+ if (i)+ {+ for (j = i; j <4; j++)+ {char_array_4[j] = 0;}- for (j = 0; j < 4; j++) {+ for (j = 0; j <4; j++)+ {char_array_4[j] = base64_chars.find(char_array_4[j]);}@@ -203,7 +541,8 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_strchar_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];- for (j = 0; j < i - 1; j++) {+ for (j = 0; (j < i - 1); j++)+ {ret.push_back(char_array_3[j]);}}@@ -215,7 +554,8 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_str// random string / id//-static std::string random_string() {+static std::string random_string()+{static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");std::random_device rd;@@ -230,389 +570,9 @@ static std::string random_string() {return result;}-static std::string gen_chatcmplid() {+static std::string gen_chatcmplid()+{std::stringstream chatcmplid;chatcmplid << "chatcmpl-" << random_string();-return chatcmplid.str();}--//-// other common utils-//--static size_t common_part(const std::vector<llama_token> & a, const std::vector<llama_token> & b) {- size_t i;- for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {}-- return i;-}--static bool ends_with(const std::string & str, const std::string & suffix) {- return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);-}--static size_t find_partial_stop_string(const std::string &stop, const std::string &text) {- if (!text.empty() && !stop.empty()) {- const char text_last_char = text.back();- for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) {- if (stop[char_index] == text_last_char) {- const std::string current_partial = stop.substr(0, char_index + 1);- if (ends_with(text, current_partial)) {- return text.size() - char_index - 1;- }- }- }- }-- return std::string::npos;-}--// TODO: reuse llama_detokenize-template <class Iter>-static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) {- std::string ret;- for (; begin != end; ++begin) {- ret += llama_token_to_piece(ctx, *begin);- }-- return ret;-}--// format incomplete utf-8 multibyte character for output-static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {- std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token);-- // if the size is 1 and first bit is 1, meaning it's a partial character- // (size > 1 meaning it's already a known token)- if (out.size() == 1 && (out[0] & 0x80) == 0x80) {- std::stringstream ss;- ss << std::hex << (out[0] & 0xff);- std::string res(ss.str());- out = "byte: \\x" + res;- }-- return out;-}--struct completion_token_output {- llama_token tok;- std::string text_to_send;-- struct token_prob {- llama_token tok;- float prob;- };-- std::vector<token_prob> probs;-};--// convert a vector of completion_token_output to json-static json probs_vector_to_json(const llama_context * ctx, const std::vector<completion_token_output> & probs) {- json out = json::array();-- for (const auto & prob : probs) {- json probs_for_token = json::array();-- for (const auto & p : prob.probs) {- const std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);- probs_for_token.push_back(json {- {"tok_str", tok_str},- {"prob", p.prob},- });- }-- const std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);- out.push_back(json {- {"content", tok_str},- {"probs", probs_for_token},- });- }-- return out;-}--//-// OAI utils-//--static json oaicompat_completion_params_parse(- const struct llama_model * model,- const json & body, /* openai api json semantics */- const std::string & chat_template) {- json llama_params;-- llama_params["__oaicompat"] = true;-- // Map OpenAI parameters to llama.cpp parameters- //- // For parameters that are defined by the OpenAI documentation (e.g.- // temperature), we explicitly specify OpenAI's intended default; we- // need to do that because sometimes OpenAI disagrees with llama.cpp- //- // https://platform.openai.com/docs/api-reference/chat/create- llama_sampling_params default_sparams;- llama_params["model"] = json_value(body, "model", std::string("unknown"));- llama_params["prompt"] = format_chat(model, chat_template, body["messages"]);- llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);- llama_params["temperature"] = json_value(body, "temperature", 0.0);- llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);- llama_params["top_p"] = json_value(body, "top_p", 1.0);- llama_params["n_predict"] = json_value(body, "max_tokens", -1);- llama_params["logit_bias"] = json_value(body, "logit_bias", json::object());- llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);- llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);- llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);- llama_params["stream"] = json_value(body, "stream", false);- llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);- llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);- llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);- llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);- llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);- llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);- llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);- llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);- llama_params["n_keep"] = json_value(body, "n_keep", 0);-- if (body.contains("grammar")) {- llama_params["grammar"] = json_value(body, "grammar", json::object());- }-- if (body.contains("response_format")) {- auto response_format = json_value(body, "response_format", json::object());- if (response_format.contains("type")) {- if (response_format["type"] == "json_object") {- llama_params["json_schema"] = json_value(response_format, "schema", json::object());- } else {- throw std::runtime_error("response_format type not supported: " + response_format["type"].dump());- }- }- }-- // Handle 'stop' field- if (body.contains("stop") && body["stop"].is_string()) {- llama_params["stop"] = json::array({body["stop"].get<std::string>()});- } else {- llama_params["stop"] = json_value(body, "stop", json::array());- }-- // Ensure there is ChatML-specific end sequence among stop words- llama_params["stop"].push_back("<|im_end|>");-- return llama_params;-}--static json format_final_response_oaicompat(const json & request, json result, const std::string & completion_id, bool streaming = false) {- bool stopped_word = result.count("stopped_word") != 0;- bool stopped_eos = json_value(result, "stopped_eos", false);- int num_tokens_predicted = json_value(result, "tokens_predicted", 0);- int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);- std::string content = json_value(result, "content", std::string(""));-- std::string finish_reason = "length";- if (stopped_word || stopped_eos) {- finish_reason = "stop";- }-- json choices =- streaming ? json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"delta", json::object()}}})- : json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"message", json{{"content", content},- {"role", "assistant"}}}}});-- std::time_t t = std::time(0);-- json res = json {- {"choices", choices},- {"created", t},- {"model",- json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},- {"object", streaming ? "chat.completion.chunk" : "chat.completion"},- {"usage", json {- {"completion_tokens", num_tokens_predicted},- {"prompt_tokens", num_prompt_tokens},- {"total_tokens", num_tokens_predicted + num_prompt_tokens}- }},- {"id", completion_id}- };-- if (server_verbose) {- res["__verbose"] = result;- }-- if (result.contains("completion_probabilities")) {- res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());- }-- return res;-}--// return value is vector as there is one case where we might need to generate two responses-static std::vector<json> format_partial_response_oaicompat(json result, const std::string & completion_id) {- if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {- return std::vector<json>({result});- }-- bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;- std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));-- bool stopped_word = json_value(result, "stopped_word", false);- bool stopped_eos = json_value(result, "stopped_eos", false);- bool stopped_limit = json_value(result, "stopped_limit", false);- std::string content = json_value(result, "content", std::string(""));-- std::string finish_reason;- if (stopped_word || stopped_eos) {- finish_reason = "stop";- }- if (stopped_limit) {- finish_reason = "length";- }-- std::time_t t = std::time(0);-- json choices;-- if (!finish_reason.empty()) {- choices = json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"delta", json::object()}}});- } else {- if (first) {- if (content.empty()) {- choices = json::array({json{{"finish_reason", nullptr},- {"index", 0},- {"delta", json{{"role", "assistant"}}}}});- } else {- // We have to send this as two updates to conform to openai behavior- json initial_ret = json{{"choices", json::array({json{- {"finish_reason", nullptr},- {"index", 0},- {"delta", json{- {"role", "assistant"}- }}}})},- {"created", t},- {"id", completion_id},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- json second_ret = json{- {"choices", json::array({json{{"finish_reason", nullptr},- {"index", 0},- {"delta", json{- {"content", content}}}- }})},- {"created", t},- {"id", completion_id},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- return std::vector<json>({initial_ret, second_ret});- }- } else {- // Some idiosyncrasy in task processing logic makes several trailing calls- // with empty content, we ignore these at the calee site.- if (content.empty()) {- return std::vector<json>({json::object()});- }-- choices = json::array({json{- {"finish_reason", nullptr},- {"index", 0},- {"delta",- json{- {"content", content},- }},- }});- }- }-- json ret = json {- {"choices", choices},- {"created", t},- {"id", completion_id},- {"model", modelname},- {"object", "chat.completion.chunk"}- };-- return std::vector<json>({ret});-}--static json format_embeddings_response_oaicompat(const json & request, const json & embeddings) {- json data = json::array();- int i = 0;- for (auto & elem : embeddings) {- data.push_back(json{- {"embedding", json_value(elem, "embedding", json::array())},- {"index", i++},- {"object", "embedding"}- });- }-- json res = json {- {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},- {"object", "list"},- {"usage", json {- {"prompt_tokens", 0},- {"total_tokens", 0}- }},- {"data", data}- };-- return res;-}--static json format_tokenizer_response(const std::vector<llama_token> & tokens) {- return json {- {"tokens", tokens}- };-}--static json format_detokenized_response(const std::string & content) {- return json {- {"content", content}- };-}--static json format_error_response(const std::string & message, const enum error_type type) {- std::string type_str;- int code = 500;- switch (type) {- case ERROR_TYPE_INVALID_REQUEST:- type_str = "invalid_request_error";- code = 400;- break;- case ERROR_TYPE_AUTHENTICATION:- type_str = "authentication_error";- code = 401;- break;- case ERROR_TYPE_NOT_FOUND:- type_str = "not_found_error";- code = 404;- break;- case ERROR_TYPE_SERVER:- type_str = "server_error";- code = 500;- break;- case ERROR_TYPE_PERMISSION:- type_str = "permission_error";- code = 403;- break;- case ERROR_TYPE_NOT_SUPPORTED:- type_str = "not_supported_error";- code = 501;- break;- case ERROR_TYPE_UNAVAILABLE:- type_str = "unavailable_error";- code = 503;- break;- }- return json {- {"code", code},- {"message", message},- {"type", type_str},- };-}diff --git a/llamafile/kernel.h b/llamafile/kernel.hindex 06777161..5bdca973 100644--- a/llamafile/kernel.h+++ b/llamafile/kernel.h@@ -4,12 +4,11 @@int ytiles = (m - m0) / RM; \int xtiles = (n - n0) / RN; \int tiles = ytiles * xtiles; \- double duty = (double)tiles / nth; \+ int duty = (tiles + nth - 1) / nth; \if (duty < 1) \duty = 1; \- double spot = duty * ith + .5; \- int end = spot + duty; \- int start = spot; \+ int start = duty * ith; \+ int end = start + duty; \if (end > tiles) \end = tiles; \for (int job = start; job < end; ++job) { \diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 10b539f0..3fde8dd0 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -16,13 +16,9 @@// limitations under the License.#ifdef __x86_64__-+#include "sgemm.h"#include "llamafile.h"-#include <cosmo.h>-#include <stdio.h>--#include "sgemm.h"bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {diff --git a/llamafile/sgemm_test.cpp b/llamafile/sgemm_test.cppindex 03978f51..e633f725 100644--- a/llamafile/sgemm_test.cpp+++ b/llamafile/sgemm_test.cpp@@ -36,22 +36,23 @@ float numba(void) { // (-1,1)return float01(rand32()) * 2 - 1;}-template <typename T> void randomize(T *A, int n) {- for (int i = 0; i < n; ++i)- A[i] = numba();-}-template <typename T> void broadcast(T *A, int n, T x) {for (int i = 0; i < n; ++i)A[i] = x;}+template <typename T> void randomize(int m, int n, T *A, int lda) {+ for (int j = 0; j < n; ++j)+ for (int i = 0; i < m; ++i)+ A[lda * j + i] = numba();+}+int main(int argc, char *argv[]) {float tolerance = 1e-5;- int n = 32;- int m = 32;- int k = 1024;- int l = 0;+ int m = 510;+ int n = 513;+ int k = 512;+ int l = 3;int lda = k + l;int ldb = k + l;int ldc = m + l;@@ -59,11 +60,13 @@ int main(int argc, char *argv[]) {float *B = new float[ldb * n];float *C = new float[ldc * n];float *G = new float[ldc * n];- randomize(A, lda * m);- randomize(B, ldb * n);+ broadcast(A, lda * m, NAN);+ broadcast(B, ldb * n, NAN);broadcast(C, ldc * n, NAN);broadcast(G, ldc * n, NAN);- gemm<double>(true, false, m, n, k, 1.f, A, lda, B, ldb, 0.f, G, ldc);+ randomize(k, m, A, lda);+ randomize(k, n, B, ldb);+ gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc);if (!llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32))return 1;