LLM inference in C/C++
C++ ggml
cli: add --output option (#25484)
Xuan-Son Nguyen3de7dd4c8f5d
diff --git a/common/arg.cpp b/common/arg.cppindex 202ebcb0f0..8a78291658 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2849,7 +2849,7 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.out_file = value;}).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_FINETUNE,- LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS}));+ LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS, LLAMA_EXAMPLE_CLI}));add_opt(common_arg({"-ofreq", "--output-frequency"}, "N",string_format("output the imatrix every N iterations (default: %d)", params.n_out_freq),diff --git a/tools/cli/cli-context.cpp b/tools/cli/cli-context.cppindex 78b1b102f1..74d60eb19d 100644--- a/tools/cli/cli-context.cpp+++ b/tools/cli/cli-context.cpp@@ -162,6 +162,14 @@ bool cli_context::init() {fetch_server_props();+ if (!params.out_file.empty()) {+ output_file.emplace(params.out_file);+ if (!output_file->is_open()) {+ ui::show_error(string_format("failed to open output file '%s'", params.out_file.c_str()));+ return false;+ }+ }+return true;}@@ -323,7 +331,14 @@ bool cli_context::stage_media_file(const std::string & fname, const std::stringreturn true;}-bool cli_context::generate_completion(std::string & assistant_content, cli_timings & timings) {+void cli_context::write_output_file(const std::string & content) {+ if (output_file) {+ (*output_file) << content;+ output_file->flush();+ }+}++bool cli_context::generate_completion(generated_content & content_out, cli_timings & timings) {json body = {{"messages", impl->messages},{"stream", true},@@ -364,13 +379,14 @@ bool cli_context::generate_completion(std::string & assistant_content, cli_timinif (delta.contains("reasoning_content") && delta.at("reasoning_content").is_string()) {const std::string text = delta.at("reasoning_content").get<std::string>();if (!text.empty()) {+ content_out.reasoning += text;a.push(ui::ASSISTANT_DISPLAY_MODE_REASONING, text);}}if (delta.contains("content") && delta.at("content").is_string()) {const std::string text = delta.at("content").get<std::string>();if (!text.empty()) {- assistant_content += text;+ content_out.content += text;a.push(ui::ASSISTANT_DISPLAY_MODE_CONTENT, text);}}@@ -520,10 +536,12 @@ int cli_context::run() {continue;}ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));+ write_output_file(string_format("User: Added media: %s\n", fname.c_str()));continue;} else if (string_starts_with(buffer, "/read ")) {std::string fname = string_strip(buffer.substr(6));add_text_file(fname);+ write_output_file(string_format("User: Added text file: %s\n", fname.c_str()));continue;} else if (string_starts_with(buffer, "/glob ")) {std::error_code ec;@@ -568,9 +586,11 @@ int cli_context::run() {continue;}- if (!add_text_file((rel_path / rel).string())) {+ const std::string full_path = (curdir / rel).string();+ if (!add_text_file(full_path)) {continue;}+ write_output_file(string_format("User: Added text file: %s\n", full_path.c_str()));if (++count >= FILE_GLOB_MAX_RESULTS) {ui::show_error(string_format("Maximum number of globbed files allowed (%zu) reached.", FILE_GLOB_MAX_RESULTS));@@ -586,16 +606,34 @@ int cli_context::run() {// generate responseif (add_user_msg) {push_user_message(cur_msg);+ write_output_file(string_format("User:\n%s\n\n", cur_msg.c_str()));cur_msg.clear();}+cli_timings timings;- std::string assistant_content;- generate_completion(assistant_content, timings);+ generated_content content;+ generate_completion(content, timings);+impl->messages.push_back({{"role", "assistant"},- {"content", assistant_content}+ {"content", content.content}});+ if (output_file) {+ std::string out_content = "Assistant:\n";+ if (!content.reasoning.empty()) {+ out_content += "[Start thinking]\n\n";+ out_content += content.reasoning;+ out_content += "[End thinking]\n\n";+ }+ out_content += content.content;+ if (!out_content.empty() && out_content.back() != '\n') {+ out_content += "\n";+ }+ out_content += "\n";+ write_output_file(out_content);+ }+if (params.show_timings) {ui::show_info(string_format("\n[ Prompt: %.1f t/s | Generation: %.1f t/s ]",@@ -619,4 +657,8 @@ void cli_context::shutdown() {server->stop();server.reset();}+ if (output_file) {+ output_file->close();+ output_file.reset();+ }}diff --git a/tools/cli/cli-context.h b/tools/cli/cli-context.hindex 79694fb6c0..15ce4efee6 100644--- a/tools/cli/cli-context.h+++ b/tools/cli/cli-context.h@@ -9,6 +9,7 @@#include <memory>#include <optional>#include <string>+#include <fstream>struct cli_timings {double prompt_per_second = 0.0;@@ -32,6 +33,8 @@ struct cli_context {bool has_audio = false;bool has_video = false;+ std::optional<std::ofstream> output_file;+cli_context(const common_params & params);~cli_context();@@ -49,7 +52,11 @@ struct cli_context {static std::atomic<bool> & interrupted();private:- bool generate_completion(std::string & assistant_content, cli_timings & timings);+ struct generated_content {+ std::string reasoning;+ std::string content;+ };+ bool generate_completion(generated_content & content_out, cli_timings & timings);void fetch_server_props();void add_system_prompt();void push_user_message(const std::string & text);@@ -62,5 +69,8 @@ private:// "image", "audio", "video"; returns false if the file cannot be readbool stage_media_file(const std::string & fname, const std::string & type);+ // no-op if output file is not set+ void write_output_file(const std::string & content);+std::unique_ptr<cli_context_impl> impl;};