Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add code for sending chunked responses
Justine Tunneycf00c15f9901
diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cppindex 7a5f016c..50fc630e 100644--- a/llamafile/server/client.cpp+++ b/llamafile/server/client.cpp@@ -261,28 +261,36 @@ Client::transport()return dispatch();}-void-Client::begin_response()-{- cleanup();- pthread_testcancel();- should_send_error_if_canceled = false;-}-+// sends http error response message and body in one shot+//+// after this function is called, the handler must return control.+//+// @param code must be a number between 400 and 999+// @param reason must be a sanitized http token, or null for defaultboolClient::send_error(int code, const char* reason){- begin_response();if (!reason)reason = GetHttpReason(code);SLOG("error %d %s", code, reason);- char* p = start_response(obuf.p, code, reason);+ char* p = append_http_response_message(obuf.p, code, reason);return send_response(obuf.p, p, string(reason) + "\r\n");}+// appends start of http response message to `p`+//+// after this function is called, more header lines may be appended.+// afterwards, either send_response() or send_response_start() should be+// called to transmit the message over the wire.+//+// @param p is a page guarded buffer+// @param code must be a number between 200 and 999+// @param reason must be a sanitized http token, or null for default+// @return p + len(appended content)char*-Client::start_response(char* p, int code, const char* reason)+Client::append_http_response_message(char* p, int code, const char* reason){+ // generate http message starting line*p++ = 'H';*p++ = 'T';*p++ = 'T';@@ -301,14 +309,9 @@ Client::start_response(char* p, int code, const char* reason)p = stpcpy(p, reason);*p++ = '\r';*p++ = '\n';- p = stpcpy(p, STANDARD_RESPONSE_HEADERS);- return p;-}-bool-Client::send_response(char* p0, char* p, string_view content)-{- begin_response();+ // append standard headers+ p = stpcpy(p, STANDARD_RESPONSE_HEADERS);// append date headertm tm;@@ -326,6 +329,23 @@ Client::send_response(char* p0, char* p, string_view content)if (close_connection)p = stpcpy(p, "Connection: close\r\n");+ return p;+}++// sends http response message and body in one shot+//+// after this function is called, the handler must return control.+//+// @param p0 points to start of http response message+// @param p points to end of http response message headers; we assume+// that we can keep appending to `p` which must be page guarded+bool+Client::send_response(char* p0, char* p, string_view content)+{+ cleanup();+ pthread_testcancel();+ should_send_error_if_canceled = false;+// append content lengthp = stpcpy(p, "Content-Length: ");p = FormatInt64(p, content.size());@@ -339,6 +359,114 @@ Client::send_response(char* p0, char* p, string_view content)return send2(string_view(p0, p - p0), content);}+// sends http response message, but not its body.+//+// after this function is called, send_response_chunk() may be called to+// stream individual pieces of the response body. when you're done, you+// must call send_response_finish() to complete the response.+//+// here's an example handler that uses this:+//+// bool+// Client::fun()+// {+// char* message = obuf.p;+// char* p = append_http_response_message(message, 200);+// if (!send_response_start(message, p))+// return false;+// sleep(1);+// if (!send_response_chunk("hello\r\n"))+// return false;+// sleep(1);+// if (!send_response_chunk("world\r\n"))+// return false;+// sleep(1);+// return send_response_finish();+// }+//+// @param p0 points to start of http response message+// @param p points to end of http response message headers; we assume+// that we can keep appending to `p` which must be page guarded+bool+Client::send_response_start(char* p0, char* p)+{+ // use chunked transfer encoding if http/1.1+ if (msg.version >= 11)+ p = stpcpy(p, "Transfer-Encoding: chunked\r\n");++ // finish message+ *p++ = '\r';+ *p++ = '\n';++ return send(string_view(p0, p - p0));+}++// finishes sending chunked http response body.+//+// once you are finished sending chunks, call send_response_finish().+bool+Client::send_response_chunk(const string_view content)+{+ // don't encode chunk boundaries for simple http client+ // it will need to rely on read() doing the right thing+ if (msg.version < 11)+ return send(content);++ // sent in three pieces+ iovec iov[3];+ size_t bytes = 0;++ // 1. send "%zx\r\n" % (len(content))+ char start[32];+ char* p = start;+ p = FormatHex64(p, content.size(), 0);+ *p++ = '\r';+ *p++ = '\n';+ iov[0].iov_base = start;+ iov[0].iov_len = p - start;+ bytes += iov[0].iov_len;++ // 2. send content+ iov[1].iov_base = (void*)content.data();+ iov[1].iov_len = content.size();+ bytes += iov[1].iov_len;++ // 3. send newline+ iov[2].iov_base = (void*)"\r\n";+ iov[2].iov_len = 2;+ bytes += iov[2].iov_len;++ // perform send system call+ ssize_t sent;+ if ((sent = writev(fd, iov, 3)) != bytes) {+ if (sent == -1 && errno != EAGAIN && errno != ECONNRESET)+ SLOG("writev failed %m");+ return false;+ }+ return true;+}++// finishes sending chunked http response body.+//+// after this function is called, the handler must return control.+bool+Client::send_response_finish()+{+ cleanup();++ // don't encode chunk boundaries for simple http client+ // it will need to rely on read() doing the right thing+ if (msg.version < 11)+ return true;++ // send terminating chunk+ return send("0\r\n\r\n");+}++// writes raw data to socket+//+// consider using the higher level methods like send_error(),+// send_response(), send_response_start(), etc.boolClient::send(const string_view s){@@ -351,6 +479,10 @@ Client::send(const string_view s)return true;}+// writes two pieces of raw data to socket in single system call+//+// consider using the higher level methods like send_error(),+// send_response(), send_response_start(), etc.boolClient::send2(const string_view s1, const string_view s2){@@ -434,16 +566,9 @@ Client::read_payload()params_memory =ParseParams(payload.data(), payload.size(), &url.params);}-return true;}-static void-cancel_http_request(void* arg)-{- Client* client = (Client*)arg;-}-boolClient::dispatch(){diff --git a/llamafile/server/client.h b/llamafile/server/client.hindex 407dc349..baef96cb 100644--- a/llamafile/server/client.h+++ b/llamafile/server/client.h@@ -81,16 +81,19 @@ struct Clientbool read_request() __wur;bool read_content() __wur;bool send_continue() __wur;- void begin_response();bool send(const ctl::string_view) __wur;void defer_cleanup(void (*)(void*), void*);bool send_error(int, const char* = nullptr);- char* start_response(char*, int, const char* = nullptr);+ char* append_http_response_message(char*, int, const char* = nullptr);bool send_response(char*, char*, const ctl::string_view) __wur;+ bool send_response_start(char*, char*) __wur;+ bool send_response_chunk(const ctl::string_view) __wur;+ bool send_response_finish() __wur;bool send2(const ctl::string_view, const ctl::string_view) __wur;char* append_header(const ctl::string_view, const ctl::string_view);bool has_at_most_this_element(int, const ctl::string_view);ctl::string_view get_header(const ctl::string_view&);+ bool fun() __wur;ctl::string_view path();ctl::optional<ctl::string_view> param(ctl::string_view);diff --git a/llamafile/server/completion.cpp b/llamafile/server/completion.cppindex 398594ed..1ecc33b8 100644--- a/llamafile/server/completion.cpp+++ b/llamafile/server/completion.cpp@@ -311,7 +311,7 @@ Client::completion()// send responsechar* headers = p;- p = start_response(p, 200);+ p = append_http_response_message(p, 200);p = stpcpy(p, "Content-Type: application/json\r\n");p = stpcpy(p, "X-Wall-Micros: ");p = FormatInt64(p, wall_us);diff --git a/llamafile/server/embedding.cpp b/llamafile/server/embedding.cppindex ae9bd46d..b7412940 100644--- a/llamafile/server/embedding.cpp+++ b/llamafile/server/embedding.cpp@@ -323,7 +323,7 @@ Client::embedding()// send responsechar* headers = p;- p = start_response(p, 200);+ p = append_http_response_message(p, 200);p = stpcpy(p, "Content-Type: application/json\r\n");p = stpcpy(p, "X-Wall-Micros: ");p = FormatInt64(p, wall_us);diff --git a/llamafile/server/tokenize.cpp b/llamafile/server/tokenize.cppindex ef7b96bd..38f0dd08 100644--- a/llamafile/server/tokenize.cpp+++ b/llamafile/server/tokenize.cpp@@ -164,7 +164,7 @@ Client::tokenize()// send responsechar* headers = p;- p = start_response(p, 200);+ p = append_http_response_message(p, 200);p = stpcpy(p, "Content-Type: application/json\r\n");p = stpcpy(p, "X-Wall-Micros: ");p = FormatInt64(p, wall_us);