LLM inference in C/C++
C++ ggml
server: add get_info tool (#26522)
Xuan-Son Nguyen99111b19ce48
diff --git a/common/arg.cpp b/common/arg.cppindex 305938fcb2..b75f4f05f0 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2582,7 +2582,7 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.mtmd_batch_max_tokens = value;}).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));- if (llama_supports_rpc()) {+ if (params.is_gen_docs || llama_supports_rpc()) {add_opt(common_arg({"--rpc"}, "SERVERS","comma-separated list of RPC servers (host:port)",@@ -3331,7 +3331,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex{"--tools"}, "TOOL1,TOOL2,...","experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n""specify \"all\" to enable all tools\n"- "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"+ "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n""note: for security reasons, this will limit --cors-origins to localhost by default",[](common_params & params, const std::string & value) {params.server_tools = parse_csv_row(value);diff --git a/common/common.h b/common/common.hindex 919c0ea103..25dac86d8a 100644--- a/common/common.h+++ b/common/common.h@@ -739,6 +739,8 @@ struct common_params {llama_progress_callback load_progress_callback = NULL;void * load_progress_callback_user_data = NULL;bool no_alloc = false; // Don't allocate model buffers++ bool is_gen_docs = false; // whether we are running inside llama-gen-docs};// call once at the start of a program if it uses libcommondiff --git a/examples/gen-docs/gen-docs.cpp b/examples/gen-docs/gen-docs.cppindex baf61bf27b..114416719c 100644--- a/examples/gen-docs/gen-docs.cpp+++ b/examples/gen-docs/gen-docs.cpp@@ -70,6 +70,8 @@ static void write_table(std::ostringstream & ss, std::vector<common_arg *> & optstatic void write_help(std::ostringstream & ss, const md_file & md) {common_params params;+ params.is_gen_docs = true;+auto ctx_arg = common_params_parser_init(params, md.ex);std::vector<common_arg *> common_options;diff --git a/tools/server/README.md b/tools/server/README.mdindex f45c018972..a0956f9e65 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -198,7 +198,9 @@ For the full list of features, please refer to [server's changelog](https://gith| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) || `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) || `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |-| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |+| `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) |+| `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) || `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) || `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) || `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) |diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cppindex 4a6c5ed442..984bb478ea 100644--- a/tools/server/server-tools.cpp+++ b/tools/server/server-tools.cpp@@ -1090,6 +1090,56 @@ struct server_tool_get_datetime : server_tool {}};+//+// get_info: returns runtime info (OS name/version and cwd)+//++struct server_tool_get_info : server_tool {+ server_tool_get_info() {+ name = "get_info";+ display_name = "Get Runtime Info";+ permission_write = false;+ }++ json get_definition() const override {+ return {+ {"type", "function"},+ {"function", {+ {"name", name},+ {"description", "Returns runtime info: the OS name/version and the current working directory"},+ {"parameters", {+ {"type", "object"},+ {"properties", json::object()},+ }},+ }},+ };+ }++ json invoke(json params, server_tool::stream *) const override {+ auto io = make_tools_io(params);++#ifdef _WIN32+ auto res = io->run({"cmd", "/c", "ver"}, 4096, 5);+#else+ auto res = io->run({"uname", "-a"}, 4096, 5);+#endif+ // "ver" prints a blank line before the version, so the output is stripped on both ends;+ // a failed spawn or a timeout leaves a diagnostic in res.output, which is not an OS name+ std::string os_info = res.exit_code == 0 && !res.timed_out ? string_strip(res.output) : "unknown";++ std::string cwd = json_value(params, "cwd", std::string());+ if (cwd.empty()) {+ std::error_code ec;+ cwd = fs::current_path(ec).string();+ }++ return {+ {"os", os_info},+ {"cwd", cwd},+ };+ }+};+struct server_tool_stream_result : server_task_result {std::string chunk;bool done = false;@@ -1199,6 +1249,7 @@ static std::vector<std::unique_ptr<server_tool>> build_tools() {tools.push_back(std::make_unique<server_tool_write_file>());tools.push_back(std::make_unique<server_tool_edit_file>());tools.push_back(std::make_unique<server_tool_get_datetime>());+ tools.push_back(std::make_unique<server_tool_get_info>());return tools;}