Skip to content

LLM inference in C/C++

C++ ggml

server: log prompts to directory (#22031)

jacekpoplawski1e912561dd3e
diff --git a/common/arg.cpp b/common/arg.cpp
index df506d2d0e..55795d357d 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -3333,6 +3333,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
common_log_set_file(common_log_main(), value.c_str());
}
).set_env("LLAMA_ARG_LOG_FILE"));
+ add_opt(common_arg(
+ {"--log-prompts-dir"}, "PATH",
+ "Log prompts to directory (only used for debugging, default: disabled)",
+ [](common_params & params, const std::string & value) {
+ params.path_prompts_log_dir = value;
+ }
+ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
add_opt(common_arg(
{"--log-colors"}, "[on|off|auto]",
"Set colored logging ('on', 'off', or 'auto', default: 'auto')\n"
diff --git a/common/common.h b/common/common.h
index b732a2087d..4864186f62 100644
--- a/common/common.h
+++ b/common/common.h
@@ -489,6 +489,7 @@ struct common_params {
std::string input_prefix = ""; // string to prefix user inputs with // NOLINT
std::string input_suffix = ""; // string to suffix user inputs with // NOLINT
std::string logits_file = ""; // file for saving *all* logits // NOLINT
+ std::string path_prompts_log_dir = ""; // directory with logged prompts // NOLINT
// llama-debug specific options
std::string logits_output_dir = "data"; // directory for saving logits output files // NOLINT
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index a7c4f7b56e..bdfa517180 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -27,6 +27,7 @@
#include <memory>
#include <filesystem>
#include <utility>
+#include <fstream>
// fix problem with std::min and std::max
#if defined(_WIN32)
@@ -3719,6 +3720,16 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
// TODO: this log can become very long, put it behind a flag or think about a more compact format
//SRV_DBG("Prompt: %s\n", prompt.is_string() ? prompt.get<std::string>().c_str() : prompt.dump(2).c_str());
+ if (!params.path_prompts_log_dir.empty()) {
+ const auto file_path = std::filesystem::path(params.path_prompts_log_dir) / string_format("%012" PRId64 ".txt", ggml_time_ms());
+ std::ofstream f(file_path);
+ if (f) {
+ f << (prompt.is_string() ? prompt.get<std::string>().c_str() : prompt.dump(2).c_str());
+ } else {
+ SRV_ERR("failed to create %s\n", file_path.string().c_str());
+ }
+ }
+
// process prompt
std::vector<server_tokens> inputs;