LLM inference in C/C++
C++ ggml
log, server: self contained colors, split child commands from logs in router mode (#29895)
Pascala7fb71fab83b
diff --git a/common/common.cpp b/common/common.cppindex 0907765ecf..463137e5ac 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1065,6 +1065,20 @@ bool tty_can_use_colors() {return common_is_tty(stdout) || common_is_tty(stderr);}+bool tty_enable_ansi() {+#if defined(_WIN32)+ // a Windows console renders ANSI sequences only in virtual terminal mode, pipes and files take them as is+ for (DWORD id : { STD_OUTPUT_HANDLE, STD_ERROR_HANDLE }) {+ HANDLE h = GetStdHandle(id);+ DWORD mode = 0;+ if (GetConsoleMode(h, &mode) && !SetConsoleMode(h, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) {+ return false;+ }+ }+#endif+ return true;+}+//// Model utils//diff --git a/common/common.h b/common/common.hindex fa2ffd9e1a..e1ef70a9e4 100644--- a/common/common.h+++ b/common/common.h@@ -940,6 +940,7 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat// Auto-detect if colors can be enabled based on terminal and environmentbool tty_can_use_colors();+bool tty_enable_ansi(); // false when stdout or stderr is a console that cannot render ANSI sequences// Check if the given file is attached to a terminalbool common_is_tty(FILE * file);diff --git a/common/log.cpp b/common/log.cppindex b60e8d2687..86c98e8e63 100644--- a/common/log.cpp+++ b/common/log.cpp@@ -144,12 +144,16 @@ struct common_log_entry {}}- fprintf(fcur, "%s", msg.data());+ // the reset goes before the trailing newlines, so that every line carries its own colors+ const bool reset = level == GGML_LOG_LEVEL_WARN || level == GGML_LOG_LEVEL_ERROR || level == GGML_LOG_LEVEL_DEBUG;- if (level == GGML_LOG_LEVEL_WARN || level == GGML_LOG_LEVEL_ERROR || level == GGML_LOG_LEVEL_DEBUG) {- fprintf(fcur, "%s", g_col[COMMON_LOG_COL_DEFAULT]);+ size_t end = strlen(msg.data());+ while (end > 0 && msg[end - 1] == '\n') {+ end--;}+ fprintf(fcur, "%.*s%s%s", (int) end, msg.data(), reset ? g_col[COMMON_LOG_COL_DEFAULT] : "", msg.data() + end);+fflush(fcur);}};@@ -158,6 +162,7 @@ struct common_log {// default capacitycommon_log(size_t capacity = 512) {file = nullptr;+ colors = false;prefix = false;timestamps = false;running = false;@@ -185,6 +190,7 @@ private:FILE * file;+ bool colors;bool prefix;bool timestamps;bool running;@@ -394,10 +400,16 @@ public:resume();}+ bool get_colors() const {+ return colors;+ }+void set_colors(bool colors) {pause();- if (colors) {+ this->colors = colors && tty_enable_ansi();++ if (this->colors) {g_col[COMMON_LOG_COL_DEFAULT] = LOG_COL_DEFAULT;g_col[COMMON_LOG_COL_BOLD] = LOG_COL_BOLD;g_col[COMMON_LOG_COL_RED] = LOG_COL_RED;@@ -500,6 +512,10 @@ void common_log_set_colors(struct common_log * log, log_colors colors) {log->set_colors(true);}+bool common_log_get_colors(struct common_log * log) {+ return log->get_colors();+}+void common_log_set_prefix(struct common_log * log, bool prefix) {log->set_prefix(prefix);}diff --git a/common/log.h b/common/log.hindex e36b09463e..e9e1f17613 100644--- a/common/log.h+++ b/common/log.h@@ -93,6 +93,7 @@ void common_log_add(struct common_log * log, enum ggml_log_level level, const chvoid common_log_set_file (struct common_log * log, const char * file); // not thread-safevoid common_log_set_colors (struct common_log * log, log_colors colors); // not thread-safe+bool common_log_get_colors (struct common_log * log); // whether colors are enabledvoid common_log_set_prefix (struct common_log * log, bool prefix); // whether to output prefix to each logvoid common_log_set_timestamps(struct common_log * log, bool timestamps); // whether to output timestamps in the prefixvoid common_log_flush (struct common_log * log); // flush all pending log messagesdiff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex 2d90fcad2d..be87c45f7f 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -1885,38 +1885,71 @@ server_tokens format_prompt_rerank(// server_subproc//+FILE * server_reserve_stdout() {+ fflush(stdout);+ // the reserved stream is not inherited, grandchildren get stdout and stderr of their own+#ifdef _WIN32+ int fd = _dup(_fileno(stdout));+ GGML_ASSERT(fd >= 0);+ SetHandleInformation((HANDLE) _get_osfhandle(fd), HANDLE_FLAG_INHERIT, 0);+ _dup2(_fileno(stderr), _fileno(stdout));+ SetStdHandle(STD_OUTPUT_HANDLE, GetStdHandle(STD_ERROR_HANDLE));+ FILE * f = _fdopen(fd, "w");+#else+ int fd = fcntl(fileno(stdout), F_DUPFD_CLOEXEC, 0);+ GGML_ASSERT(fd >= 0);+ dup2(fileno(stderr), fileno(stdout));+ FILE * f = fdopen(fd, "w");+#endif+ GGML_ASSERT(f);+ return f;+}+bool server_subproc::has_output() {- if (out_handle >= 0) {- return true;- }- FILE * f = sproc.stdout_file(); // combined stdout/stderr- if (!f) {- return false;- }+ FILE * files[SERVER_SUBPROC_STREAMS] = { sproc.stdout_file(), sproc.stderr_file() };+ for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {+ if (out_handles[i] >= 0) {+ continue;+ }+ if (!files[i]) {+ return false;+ }#ifdef _WIN32- HANDLE h = (HANDLE) _get_osfhandle(_fileno(f));- if (h != INVALID_HANDLE_VALUE) {- out_handle = (intptr_t) h;- }+ HANDLE h = (HANDLE) _get_osfhandle(_fileno(files[i]));+ if (h == INVALID_HANDLE_VALUE) {+ return false;+ }+ out_handles[i] = (intptr_t) h;#else- int fd = fileno(f);- if (fd >= 0) {+ int fd = fileno(files[i]);+ if (fd < 0) {+ return false;+ }fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);- out_handle = fd;- }+ out_handles[i] = fd;#endif- return out_handle >= 0;+ }+ return true;+}++bool server_subproc::output_closed() const {+ return out_closed[SERVER_SUBPROC_STDOUT] && out_closed[SERVER_SUBPROC_STDERR];}-int server_subproc::read_output(char * buf, size_t len) {+int server_subproc::read_output(server_subproc_stream stream, char * buf, size_t len) {+ if (out_closed[stream]) {+ return -1;+ }if (!has_output()) {+ out_closed[stream] = true;return -1;}#ifdef _WIN32- HANDLE h = (HANDLE) out_handle;+ HANDLE h = (HANDLE) out_handles[stream];DWORD avail = 0;if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) {- return -1; // pipe broken, child gone+ out_closed[stream] = true; // pipe broken, child gone+ return -1;}if (avail == 0) {return 0;@@ -1924,24 +1957,23 @@ int server_subproc::read_output(char * buf, size_t len) {DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len;DWORD got = 0;if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) {+ out_closed[stream] = true;return -1;}return (int) got;#elsewhile (true) {- ssize_t r = read((int) out_handle, buf, len);+ ssize_t r = read((int) out_handles[stream], buf, len);if (r > 0) {return (int) r;}- if (r == 0) {- return -1; // EOF- }- if (errno == EINTR) {+ if (r < 0 && errno == EINTR) {continue;}- if (errno == EAGAIN || errno == EWOULDBLOCK) {+ if (r < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {return 0;}+ out_closed[stream] = true; // EOF or errorreturn -1;}#endif@@ -1979,11 +2011,15 @@ void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, s// no waitable wait exists for anonymous pipes, so poll them in 50 ms stepsbool any = false;for (size_t i = 0; i < procs.size(); i++) {- DWORD avail = 0;- if (!procs[i]->has_output() || !PeekNamedPipe((HANDLE) procs[i]->out_handle, NULL, 0, NULL, &avail, NULL) || avail > 0) {- ready[i] = true; // data or broken pipe, read_output() tells which- any = true;+ server_subproc * p = procs[i];+ ready[i] = !p->has_output();+ for (int s = 0; s < SERVER_SUBPROC_STREAMS && !ready[i]; s++) {+ DWORD avail = 0;+ if (!p->out_closed[s] && (!PeekNamedPipe((HANDLE) p->out_handles[s], NULL, 0, NULL, &avail, NULL) || avail > 0)) {+ ready[i] = true; // data or broken pipe, read_output() tells which+ }}+ any = any || ready[i];}if (!any) {int64_t step = timeout_ms < 0 ? 50 : std::min<int64_t>(timeout_ms, 50);@@ -1991,10 +2027,13 @@ void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, s}#elsestd::vector<pollfd> pfds;- pfds.reserve(procs.size() + 1);+ pfds.reserve(procs.size() * SERVER_SUBPROC_STREAMS + 1);pfds.push_back({ (int) wake_fd[0], POLLIN, 0 });for (auto * p : procs) {- pfds.push_back({ p->has_output() ? (int) p->out_handle : -1, POLLIN, 0 }); // poll() skips negative fds+ const bool open = p->has_output();+ for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {+ pfds.push_back({ open && !p->out_closed[s] ? (int) p->out_handles[s] : -1, POLLIN, 0 }); // poll() skips negative fds+ }}int timeout = timeout_ms < 0 ? -1 : (int) std::min<int64_t>(timeout_ms, std::numeric_limits<int>::max());int r = poll(pfds.data(), pfds.size(), timeout);@@ -2006,7 +2045,10 @@ void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, swhile (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {}}for (size_t i = 0; i < procs.size(); i++) {- ready[i] = pfds[i + 1].fd < 0 || pfds[i + 1].revents != 0;+ ready[i] = !procs[i]->has_output();+ for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {+ ready[i] = ready[i] || pfds[1 + i * SERVER_SUBPROC_STREAMS + s].revents != 0;+ }}#endif}diff --git a/tools/server/server-common.h b/tools/server/server-common.hindex 7e4ebed2a1..20cdeacf24 100644--- a/tools/server/server-common.h+++ b/tools/server/server-common.h@@ -639,6 +639,16 @@ struct server_pipe {}};+// a child server writes its state commands to stdout and its logs to stderr+enum server_subproc_stream {+ SERVER_SUBPROC_STDOUT,+ SERVER_SUBPROC_STDERR,+ SERVER_SUBPROC_STREAMS,+};++// gives the current stdout to the caller as a stream of its own, and sends everything else written to stdout to stderr+FILE * server_reserve_stdout();+// wrapper around common_subproc to manage a child server process// mainly used by router modestruct server_subproc {@@ -649,12 +659,15 @@ struct server_subproc {void terminate() { sproc.terminate(); }int join() { return sproc.join(); }- // true if the child's combined stdout/stderr pipe is available (call after create())+ // true if both output pipes of the child are available (call after create())bool has_output();- // non-blocking read+ // true once both output pipes are closed+ bool output_closed() const;++ // non-blocking read from one output pipe// returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken- int read_output(char * buf, size_t len);+ int read_output(server_subproc_stream stream, char * buf, size_t len);// wait until one of a set of children has output, wake() is called, or a timeout passesstruct waiter {@@ -664,7 +677,7 @@ struct server_subproc {// thread-safe; on Windows this is a no-op, wait() returns within 50 ms anywayvoid wake();- // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data (or a broken pipe)+ // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data on an open pipe (or a broken pipe)void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);private:@@ -674,5 +687,6 @@ struct server_subproc {};private:- intptr_t out_handle = -1; // fd on POSIX, HANDLE on Windows; taken lazily from sproc+ intptr_t out_handles[SERVER_SUBPROC_STREAMS] = { -1, -1 }; // fd on POSIX, HANDLE on Windows; taken lazily from sproc+ bool out_closed [SERVER_SUBPROC_STREAMS] = { false, false };};diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 1a9ccd1adc..d42b523b25 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -94,7 +94,7 @@ private:std::shared_ptr<server_subproc> proc;server_child_mode mode = SERVER_CHILD_MODE_NORMAL;int port = 0;- std::string buf; // partial line+ std::string buf[SERVER_SUBPROC_STREAMS]; // partial line of each pipebool eof = false; // output closed, waiting for the process to be reapedint64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending};@@ -147,46 +147,56 @@ private:return false;}- // read what the child wrote, forward complete lines+ // read what the child wrote, handle its commands and forward its logs, line by linevoid read_output(child_t & c) {+ for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {+ read_stream(c, (server_subproc_stream) i);+ }+ c.eof = c.proc->output_closed();+ }++ void read_stream(child_t & c, server_subproc_stream stream) {char chunk[4096];- while (!c.eof) {- int n = c.proc->read_output(chunk, sizeof(chunk));+ std::string & buf = c.buf[stream];+ bool closed = false;+ while (true) {+ int n = c.proc->read_output(stream, chunk, sizeof(chunk));if (n < 0) {- c.eof = true;+ closed = true;break;}if (n == 0) {break;}- c.buf.append(chunk, (size_t) n);+ buf.append(chunk, (size_t) n);size_t start = 0;while (true) {- size_t nl = c.buf.find('\n', start);+ size_t nl = buf.find('\n', start);if (nl == std::string::npos) {break;}- std::string line = c.buf.substr(start, nl + 1 - start);+ on_line(c, stream, buf.substr(start, nl + 1 - start));start = nl + 1;- on_line(c, line);}- c.buf.erase(0, start);- if (c.buf.size() > max_line) {- c.buf.clear(); // a child that never writes a newline must not grow this without bound+ buf.erase(0, start);+ if (buf.size() > max_line) {+ buf.clear(); // a child that never writes a newline must not grow this without bound}}- if (c.eof && !c.buf.empty()) {- on_line(c, c.buf);- c.buf.clear();+ if (closed && !buf.empty()) {+ on_line(c, stream, buf);+ buf.clear();}}- void on_line(child_t & c, const std::string & line) {- if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) {+ void on_line(child_t & c, server_subproc_stream stream, const std::string & line) {+ if (stream == SERVER_SUBPROC_STDERR) {+ LOG("[%5d] %s", c.port, line.c_str()); // forward log+ } else if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) {LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the logmodels.handle_child_state(c.name, line);} else {- LOG("[%5d] %s", c.port, line.c_str()); // forward log+ SRV_WRN("[%5d] unexpected output on the command pipe: %s", c.port, line.c_str());}}@@ -513,6 +523,8 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::strpreset.set_option(ctx_preset, "LLAMA_ARG_HOST", CHILD_ADDR);preset.set_option(ctx_preset, "LLAMA_ARG_PORT", std::to_string(port));preset.set_option(ctx_preset, "LLAMA_ARG_ALIAS", name);+ // the child output goes through the router to its terminal, so it follows the router colors+ preset.set_option(ctx_preset, "LLAMA_ARG_LOG_COLORS", common_log_get_colors(common_log_main()) ? "on" : "off");// TODO: maybe validate preset before rendering ?// render argsargs = preset.to_args(bin_path);@@ -796,11 +808,12 @@ void server_models::load_models() {inst.meta.hidden = hidden_models.count(name) > 0;}};- // update_args() injects HOST/PORT/ALIAS, so strip them before comparing presets+ // update_args() injects HOST/PORT/ALIAS/LOG_COLORS, so strip them before comparing presetsauto preset_options_for_compare = [](common_preset p) {p.unset_option("LLAMA_ARG_HOST");p.unset_option("LLAMA_ARG_PORT");p.unset_option("LLAMA_ARG_ALIAS");+ p.unset_option("LLAMA_ARG_LOG_COLORS");return p.options;};@@ -1171,9 +1184,8 @@ void server_models::load(const std::string & name, const load_options & opts) {}inst.meta.args = child_args; // save for debugging- // TODO @ngxson : maybe separate stdout and stderr in the future- // so that we can use stdout for commands and stderr for logging- int options = subprocess_option_no_window | subprocess_option_combined_stdout_stderr;+ // the child writes its commands to stdout and its logs to stderr+ int options = subprocess_option_no_window;if (!inst.subproc->sproc.create(child_args, options, child_env)) {throw std::runtime_error("failed to spawn server instance");}@@ -1673,6 +1685,18 @@ void server_models::handle_child_state(const std::string & name, const std::stri// server_child//+server_child::server_child() {+ if (is_child()) {+ cmd_out = server_reserve_stdout();+ }+}++server_child::~server_child() {+ if (cmd_out) {+ fclose(cmd_out);+ }+}+bool server_child::is_child() {const char * router_port = std::getenv("LLAMA_SERVER_ROUTER_PORT");return router_port != nullptr;@@ -1795,14 +1819,8 @@ void server_child::notify_to_router(const std::string & state, const json & payl{"payload", payload},};std::lock_guard<std::mutex> lk(mtx_stdout);- common_log_pause(common_log_main());- fflush(stdout);- // the router matches the command on a line prefix, so the leading newline- // closes whatever the logger left open on the shared pipe, down to the- // trailing color reset that carries no newline of its own- fprintf(stdout, "\n%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());- fflush(stdout);- common_log_resume(common_log_main());+ fprintf(cmd_out, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());+ fflush(cmd_out);}diff --git a/tools/server/server-models.h b/tools/server/server-models.hindex 90161bf34a..0a6999ee35 100644--- a/tools/server/server-models.h+++ b/tools/server/server-models.h@@ -320,6 +320,11 @@ struct server_child {std::mutex mtx_stdout;std::atomic<bool> is_finished_downloading = false; // set by run_download+ // in a child, keeps stdout for the commands to the router, so it is created before anything is written;+ // everything else written to stdout goes to stderr with the logs+ server_child();+ ~server_child();+// return true if the current process is a child server instancebool is_child();server_child_mode get_mode();@@ -332,6 +337,9 @@ struct server_child {// notify router server for status changes (e.g. loading, downloading, sleeping, etc.)// message will be handled by server_models::handle_child_state() on the router sidevoid notify_to_router(const std::string & state_name, const json & payload);++private:+ FILE * cmd_out = nullptr; // the stdout the router reads the commands from};struct server_models_routes {diff --git a/tools/server/server.cpp b/tools/server/server.cppindex ad538a6d63..6e2d8ff9da 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -41,6 +41,7 @@ int llama_server(int argc, char ** argv);// to be used via CLI (argc / argv are used by router mode only)int llama_server(common_params & params, int argc, char ** argv);+int llama_server(common_params & params, int argc, char ** argv, server_child & child);void llama_server_terminate();void llama_server_terminate() {if (shutdown_handler) {@@ -90,6 +91,8 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t}int llama_server(int argc, char ** argv) {+ server_child child;+std::setlocale(LC_NUMERIC, "C");#ifndef _WIN32@@ -115,12 +118,17 @@ int llama_server(int argc, char ** argv) {llama_backend_init();llama_numa_init(params.numa);- const int result = llama_server(params, argc, argv);+ const int result = llama_server(params, argc, argv, child);common_log_flush(common_log_main());return result;}int llama_server(common_params & params, int argc, char ** argv) {+ server_child child;+ return llama_server(params, argc, argv, child);+}++int llama_server(common_params & params, int argc, char ** argv, server_child & child) {bool is_run_by_cli = (argv == nullptr);common_models_handler models_handler;@@ -194,7 +202,6 @@ int llama_server(common_params & params, int argc, char ** argv) {//// register API routes- server_child child; // only used in non-router modeserver_routes routes(params, ctx_server);server_tools tools;