Skip to content

LLM inference in C/C++

C++ ggml

server: add get_info tool (#26522)

Xuan-Son Nguyen99111b19ce48
diff --git a/common/arg.cpp b/common/arg.cpp
index 305938fcb2..b75f4f05f0 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -2582,7 +2582,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.mtmd_batch_max_tokens = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
- if (llama_supports_rpc()) {
+ if (params.is_gen_docs || llama_supports_rpc()) {
add_opt(common_arg(
{"--rpc"}, "SERVERS",
"comma-separated list of RPC servers (host:port)",
@@ -3331,7 +3331,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
{"--tools"}, "TOOL1,TOOL2,...",
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
"specify \"all\" to enable all tools\n"
- "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"
+ "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n"
"note: for security reasons, this will limit --cors-origins to localhost by default",
[](common_params & params, const std::string & value) {
params.server_tools = parse_csv_row(value);
diff --git a/common/common.h b/common/common.h
index 919c0ea103..25dac86d8a 100644
--- a/common/common.h
+++ b/common/common.h
@@ -739,6 +739,8 @@ struct common_params {
llama_progress_callback load_progress_callback = NULL;
void * load_progress_callback_user_data = NULL;
bool no_alloc = false; // Don't allocate model buffers
+
+ bool is_gen_docs = false; // whether we are running inside llama-gen-docs
};
// call once at the start of a program if it uses libcommon
diff --git a/examples/gen-docs/gen-docs.cpp b/examples/gen-docs/gen-docs.cpp
index baf61bf27b..114416719c 100644
--- a/examples/gen-docs/gen-docs.cpp
+++ b/examples/gen-docs/gen-docs.cpp
@@ -70,6 +70,8 @@ static void write_table(std::ostringstream & ss, std::vector<common_arg *> & opt
static void write_help(std::ostringstream & ss, const md_file & md) {
common_params params;
+ params.is_gen_docs = true;
+
auto ctx_arg = common_params_parser_init(params, md.ex);
std::vector<common_arg *> common_options;
diff --git a/tools/server/README.md b/tools/server/README.md
index f45c018972..a0956f9e65 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -198,7 +198,9 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
| `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
-| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
+| `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) |
+| `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |
| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |
| `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) |
| `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) |
diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp
index 4a6c5ed442..984bb478ea 100644
--- a/tools/server/server-tools.cpp
+++ b/tools/server/server-tools.cpp
@@ -1090,6 +1090,56 @@ struct server_tool_get_datetime : server_tool {
}
};
+//
+// get_info: returns runtime info (OS name/version and cwd)
+//
+
+struct server_tool_get_info : server_tool {
+ server_tool_get_info() {
+ name = "get_info";
+ display_name = "Get Runtime Info";
+ permission_write = false;
+ }
+
+ json get_definition() const override {
+ return {
+ {"type", "function"},
+ {"function", {
+ {"name", name},
+ {"description", "Returns runtime info: the OS name/version and the current working directory"},
+ {"parameters", {
+ {"type", "object"},
+ {"properties", json::object()},
+ }},
+ }},
+ };
+ }
+
+ json invoke(json params, server_tool::stream *) const override {
+ auto io = make_tools_io(params);
+
+#ifdef _WIN32
+ auto res = io->run({"cmd", "/c", "ver"}, 4096, 5);
+#else
+ auto res = io->run({"uname", "-a"}, 4096, 5);
+#endif
+ // "ver" prints a blank line before the version, so the output is stripped on both ends;
+ // a failed spawn or a timeout leaves a diagnostic in res.output, which is not an OS name
+ std::string os_info = res.exit_code == 0 && !res.timed_out ? string_strip(res.output) : "unknown";
+
+ std::string cwd = json_value(params, "cwd", std::string());
+ if (cwd.empty()) {
+ std::error_code ec;
+ cwd = fs::current_path(ec).string();
+ }
+
+ return {
+ {"os", os_info},
+ {"cwd", cwd},
+ };
+ }
+};
+
struct server_tool_stream_result : server_task_result {
std::string chunk;
bool done = false;
@@ -1199,6 +1249,7 @@ static std::vector<std::unique_ptr<server_tool>> build_tools() {
tools.push_back(std::make_unique<server_tool_write_file>());
tools.push_back(std::make_unique<server_tool_edit_file>());
tools.push_back(std::make_unique<server_tool_get_datetime>());
+ tools.push_back(std::make_unique<server_tool_get_info>());
return tools;
}