Skip to content

Distribute and run LLMs with a single file.

Add code for sending chunked responses

Justine Tunneycf00c15f9901
diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cpp
index 7a5f016c..50fc630e 100644
--- a/llamafile/server/client.cpp
+++ b/llamafile/server/client.cpp
@@ -261,28 +261,36 @@ Client::transport()
return dispatch();
}
-void
-Client::begin_response()
-{
- cleanup();
- pthread_testcancel();
- should_send_error_if_canceled = false;
-}
-
+// sends http error response message and body in one shot
+//
+// after this function is called, the handler must return control.
+//
+// @param code must be a number between 400 and 999
+// @param reason must be a sanitized http token, or null for default
bool
Client::send_error(int code, const char* reason)
{
- begin_response();
if (!reason)
reason = GetHttpReason(code);
SLOG("error %d %s", code, reason);
- char* p = start_response(obuf.p, code, reason);
+ char* p = append_http_response_message(obuf.p, code, reason);
return send_response(obuf.p, p, string(reason) + "\r\n");
}
+// appends start of http response message to `p`
+//
+// after this function is called, more header lines may be appended.
+// afterwards, either send_response() or send_response_start() should be
+// called to transmit the message over the wire.
+//
+// @param p is a page guarded buffer
+// @param code must be a number between 200 and 999
+// @param reason must be a sanitized http token, or null for default
+// @return p + len(appended content)
char*
-Client::start_response(char* p, int code, const char* reason)
+Client::append_http_response_message(char* p, int code, const char* reason)
{
+ // generate http message starting line
*p++ = 'H';
*p++ = 'T';
*p++ = 'T';
@@ -301,14 +309,9 @@ Client::start_response(char* p, int code, const char* reason)
p = stpcpy(p, reason);
*p++ = '\r';
*p++ = '\n';
- p = stpcpy(p, STANDARD_RESPONSE_HEADERS);
- return p;
-}
-bool
-Client::send_response(char* p0, char* p, string_view content)
-{
- begin_response();
+ // append standard headers
+ p = stpcpy(p, STANDARD_RESPONSE_HEADERS);
// append date header
tm tm;
@@ -326,6 +329,23 @@ Client::send_response(char* p0, char* p, string_view content)
if (close_connection)
p = stpcpy(p, "Connection: close\r\n");
+ return p;
+}
+
+// sends http response message and body in one shot
+//
+// after this function is called, the handler must return control.
+//
+// @param p0 points to start of http response message
+// @param p points to end of http response message headers; we assume
+// that we can keep appending to `p` which must be page guarded
+bool
+Client::send_response(char* p0, char* p, string_view content)
+{
+ cleanup();
+ pthread_testcancel();
+ should_send_error_if_canceled = false;
+
// append content length
p = stpcpy(p, "Content-Length: ");
p = FormatInt64(p, content.size());
@@ -339,6 +359,114 @@ Client::send_response(char* p0, char* p, string_view content)
return send2(string_view(p0, p - p0), content);
}
+// sends http response message, but not its body.
+//
+// after this function is called, send_response_chunk() may be called to
+// stream individual pieces of the response body. when you're done, you
+// must call send_response_finish() to complete the response.
+//
+// here's an example handler that uses this:
+//
+// bool
+// Client::fun()
+// {
+// char* message = obuf.p;
+// char* p = append_http_response_message(message, 200);
+// if (!send_response_start(message, p))
+// return false;
+// sleep(1);
+// if (!send_response_chunk("hello\r\n"))
+// return false;
+// sleep(1);
+// if (!send_response_chunk("world\r\n"))
+// return false;
+// sleep(1);
+// return send_response_finish();
+// }
+//
+// @param p0 points to start of http response message
+// @param p points to end of http response message headers; we assume
+// that we can keep appending to `p` which must be page guarded
+bool
+Client::send_response_start(char* p0, char* p)
+{
+ // use chunked transfer encoding if http/1.1
+ if (msg.version >= 11)
+ p = stpcpy(p, "Transfer-Encoding: chunked\r\n");
+
+ // finish message
+ *p++ = '\r';
+ *p++ = '\n';
+
+ return send(string_view(p0, p - p0));
+}
+
+// finishes sending chunked http response body.
+//
+// once you are finished sending chunks, call send_response_finish().
+bool
+Client::send_response_chunk(const string_view content)
+{
+ // don't encode chunk boundaries for simple http client
+ // it will need to rely on read() doing the right thing
+ if (msg.version < 11)
+ return send(content);
+
+ // sent in three pieces
+ iovec iov[3];
+ size_t bytes = 0;
+
+ // 1. send "%zx\r\n" % (len(content))
+ char start[32];
+ char* p = start;
+ p = FormatHex64(p, content.size(), 0);
+ *p++ = '\r';
+ *p++ = '\n';
+ iov[0].iov_base = start;
+ iov[0].iov_len = p - start;
+ bytes += iov[0].iov_len;
+
+ // 2. send content
+ iov[1].iov_base = (void*)content.data();
+ iov[1].iov_len = content.size();
+ bytes += iov[1].iov_len;
+
+ // 3. send newline
+ iov[2].iov_base = (void*)"\r\n";
+ iov[2].iov_len = 2;
+ bytes += iov[2].iov_len;
+
+ // perform send system call
+ ssize_t sent;
+ if ((sent = writev(fd, iov, 3)) != bytes) {
+ if (sent == -1 && errno != EAGAIN && errno != ECONNRESET)
+ SLOG("writev failed %m");
+ return false;
+ }
+ return true;
+}
+
+// finishes sending chunked http response body.
+//
+// after this function is called, the handler must return control.
+bool
+Client::send_response_finish()
+{
+ cleanup();
+
+ // don't encode chunk boundaries for simple http client
+ // it will need to rely on read() doing the right thing
+ if (msg.version < 11)
+ return true;
+
+ // send terminating chunk
+ return send("0\r\n\r\n");
+}
+
+// writes raw data to socket
+//
+// consider using the higher level methods like send_error(),
+// send_response(), send_response_start(), etc.
bool
Client::send(const string_view s)
{
@@ -351,6 +479,10 @@ Client::send(const string_view s)
return true;
}
+// writes two pieces of raw data to socket in single system call
+//
+// consider using the higher level methods like send_error(),
+// send_response(), send_response_start(), etc.
bool
Client::send2(const string_view s1, const string_view s2)
{
@@ -434,16 +566,9 @@ Client::read_payload()
params_memory =
ParseParams(payload.data(), payload.size(), &url.params);
}
-
return true;
}
-static void
-cancel_http_request(void* arg)
-{
- Client* client = (Client*)arg;
-}
-
bool
Client::dispatch()
{
diff --git a/llamafile/server/client.h b/llamafile/server/client.h
index 407dc349..baef96cb 100644
--- a/llamafile/server/client.h
+++ b/llamafile/server/client.h
@@ -81,16 +81,19 @@ struct Client
bool read_request() __wur;
bool read_content() __wur;
bool send_continue() __wur;
- void begin_response();
bool send(const ctl::string_view) __wur;
void defer_cleanup(void (*)(void*), void*);
bool send_error(int, const char* = nullptr);
- char* start_response(char*, int, const char* = nullptr);
+ char* append_http_response_message(char*, int, const char* = nullptr);
bool send_response(char*, char*, const ctl::string_view) __wur;
+ bool send_response_start(char*, char*) __wur;
+ bool send_response_chunk(const ctl::string_view) __wur;
+ bool send_response_finish() __wur;
bool send2(const ctl::string_view, const ctl::string_view) __wur;
char* append_header(const ctl::string_view, const ctl::string_view);
bool has_at_most_this_element(int, const ctl::string_view);
ctl::string_view get_header(const ctl::string_view&);
+ bool fun() __wur;
ctl::string_view path();
ctl::optional<ctl::string_view> param(ctl::string_view);
diff --git a/llamafile/server/completion.cpp b/llamafile/server/completion.cpp
index 398594ed..1ecc33b8 100644
--- a/llamafile/server/completion.cpp
+++ b/llamafile/server/completion.cpp
@@ -311,7 +311,7 @@ Client::completion()
// send response
char* headers = p;
- p = start_response(p, 200);
+ p = append_http_response_message(p, 200);
p = stpcpy(p, "Content-Type: application/json\r\n");
p = stpcpy(p, "X-Wall-Micros: ");
p = FormatInt64(p, wall_us);
diff --git a/llamafile/server/embedding.cpp b/llamafile/server/embedding.cpp
index ae9bd46d..b7412940 100644
--- a/llamafile/server/embedding.cpp
+++ b/llamafile/server/embedding.cpp
@@ -323,7 +323,7 @@ Client::embedding()
// send response
char* headers = p;
- p = start_response(p, 200);
+ p = append_http_response_message(p, 200);
p = stpcpy(p, "Content-Type: application/json\r\n");
p = stpcpy(p, "X-Wall-Micros: ");
p = FormatInt64(p, wall_us);
diff --git a/llamafile/server/tokenize.cpp b/llamafile/server/tokenize.cpp
index ef7b96bd..38f0dd08 100644
--- a/llamafile/server/tokenize.cpp
+++ b/llamafile/server/tokenize.cpp
@@ -164,7 +164,7 @@ Client::tokenize()
// send response
char* headers = p;
- p = start_response(p, 200);
+ p = append_http_response_message(p, 200);
p = stpcpy(p, "Content-Type: application/json\r\n");
p = stpcpy(p, "X-Wall-Micros: ");
p = FormatInt64(p, wall_us);