LLM inference in C/C++
C++ ggml
server: add initial tool isolation support (via docker) (#26507)
Xuan-Son Nguyendd2c7c44710e
diff --git a/common/arg.cpp b/common/arg.cppindex da40874740..4cb853c7a4 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -3308,6 +3308,16 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.server_tools = parse_csv_row(value);}).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS"));+ add_opt(common_arg(+ {"--tools-runtime"}, "OPTION",+ "experimental: run tools in a separate runtime environment (default: none, use host environment)\n"+ "available options:\n"+ " 'docker:<image>': spin up a new Docker container and reuse it for all invocations, clean up on server exit\n"+ " 'docker-container:<id>': use an existing Docker container by ID, won't stop on server exit\n",+ [](common_params & params, const std::string & value) {+ params.server_tools_runtime = value;+ }+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME"));add_opt(common_arg({"--mcp-servers-config"}, "PATH","experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n"diff --git a/common/common.h b/common/common.hindex 2e15ec3f81..4811345f98 100644--- a/common/common.h+++ b/common/common.h@@ -655,6 +655,7 @@ struct common_params {// enable built-in toolsstd::vector<std::string> server_tools;+ std::string server_tools_runtime;// MCP server configs (Cursor-compatible JSON)std::string mcp_servers_config; // path to JSON file with MCP server definitionsdiff --git a/tools/cli/README.md b/tools/cli/README.mdindex 4d86ce7c01..640d4fee80 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -54,7 +54,6 @@| `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) || `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) |-| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) || `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |diff --git a/tools/completion/README.md b/tools/completion/README.mdindex 2abe7aaa25..e0923ea300 100644--- a/tools/completion/README.md+++ b/tools/completion/README.md@@ -137,7 +137,6 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1| `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) || `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) |-| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) || `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |diff --git a/tools/server/README-dev.md b/tools/server/README-dev.mdindex 45bcdcca76..31408f4267 100644--- a/tools/server/README-dev.md+++ b/tools/server/README-dev.md@@ -201,6 +201,7 @@ Invoke a tool call, request body is a JSON object with:Headers:- `x-tool-cwd`: optional; if set, use as the CWD for tool; this is not part of tool's params because it's meant to be set by the runtime, not the LLM itself+- `x-tool-runtime`: optional; if set, run the tool inside this isolate instead of on the host. Only `docker-container:<id>` is supported for now, using an already-running containerReturns JSON object. There are two response formats (MCP tools use the same two formats: their result content is concatenated into `plain_text_response`, and RPC or tool errors are surfaced as the `error` string):diff --git a/tools/server/README.md b/tools/server/README.mdindex 4d80f059d5..64f0b03269 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -71,7 +71,6 @@ For the full list of features, please refer to [server's changelog](https://gith| `-ctk, --cache-type-k TYPE` | KV cache data type for K<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_K) || `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) || `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) |-| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) || `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |@@ -198,6 +197,8 @@ For the full list of features, please refer to [server's changelog](https://gith| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) || `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) || `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |+| `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/> 'docker:<image>': spin up a new Docker container and reuse it for all invocations, clean up on server exit<br/> 'docker-container:<id>': use an existing Docker container by ID, won't stop on server exit<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) || `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) || `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) || `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cppindex 2fcb2a3c88..eacfbf0f74 100644--- a/tools/server/server-tools.cpp+++ b/tools/server/server-tools.cpp@@ -10,12 +10,15 @@#include <ctime>#include <atomic>#include <cstring>+#include <cstdint>#include <cstdlib>#include <algorithm>+#include <iterator>#include <unordered_set>#include <tuple>#include <functional>#include <memory>+#include <mutex>#if defined(_WIN32)# ifndef NOMINMAX@@ -127,6 +130,13 @@ static int entry_depth(const std::string & rel) {return 1 + (int) std::count(rel.begin(), rel.end(), '/');}+// directories that a listing reports but never descends into: they can be enormous+// lowercase only, the local walker case-folds a name before the lookup+static const char * const SERVER_TOOL_JUNK_DIR_NAMES[] = {+ ".git", ".svn", ".hg", "node_modules", "__pycache__",+ ".venv", "venv", "dist", "build", "target", ".cache", ".idea", ".vscode",+};+class tools_io {public:struct exec_result {@@ -165,6 +175,85 @@ public:const std::function<bool(const std::string &)> & on_chunk = nullptr) const = 0;};+// shared subprocess execution helper, used by both the local and the docker-backed tools_io implementations.+// combine_stderr=false when the raw stdout bytes must not be tainted by stderr, e.g. reading file contents.+static tools_io::exec_result run_subprocess(+ const std::vector<std::string> & args,+ size_t max_output,+ int timeout_secs,+ const std::function<bool(const std::string &)> & on_chunk,+ bool combine_stderr,+ const std::string & cwd = "") {+ tools_io::exec_result res;++ common_subproc proc;++ int options = subprocess_option_no_window+ | subprocess_option_inherit_environment+ | subprocess_option_search_user_path;+ if (combine_stderr) {+ options |= subprocess_option_combined_stdout_stderr;+ }++ if (!proc.create(args, options, {}, cwd.empty() ? nullptr : cwd.c_str())) {+ res.output = "failed to spawn process";+ return res;+ }++ std::atomic<bool> done{false};+ std::atomic<bool> timed_out{false};++ std::thread timeout_thread([&]() {+ auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_secs);+ while (!done.load()) {+ if (std::chrono::steady_clock::now() >= deadline) {+ timed_out.store(true);+ proc.terminate();+ return;+ }+ std::this_thread::sleep_for(std::chrono::milliseconds(100));+ }+ });++ FILE * f = proc.stdout_file();+ std::string output;+ bool truncated = false;+ if (f) {+ char buf[4096];+ while (fgets(buf, sizeof(buf), f) != nullptr) {+ if (!truncated) {+ size_t len = strlen(buf);+ if (output.size() + len <= max_output) {+ output.append(buf, len);+ if (on_chunk && !on_chunk(console_output_to_utf8(std::string(buf, len)))) {+ proc.terminate();+ break;+ }+ } else {+ size_t remaining = max_output - output.size();+ output.append(buf, remaining);+ if (on_chunk && remaining > 0) on_chunk(console_output_to_utf8(std::string(buf, remaining)));+ truncated = true;+ }+ }+ }+ }++ done.store(true);+ if (timeout_thread.joinable()) {+ timeout_thread.join();+ }++ res.exit_code = proc.join();++ res.output = console_output_to_utf8(output);+ res.timed_out = timed_out.load();+ if (truncated) {+ res.output += "\n[output truncated]";+ }+ return res;+}+class tools_io_basic : public tools_io {public:// cwd, if non-empty, is used to resolve relative paths and as the working directory for run()@@ -276,72 +365,7 @@ public:size_t max_output,int timeout_secs,const std::function<bool(const std::string &)> & on_chunk = nullptr) const override {- exec_result res;-- common_subproc proc;-- int options = subprocess_option_no_window- | subprocess_option_combined_stdout_stderr- | subprocess_option_inherit_environment- | subprocess_option_search_user_path;-- if (!proc.create(args, options, {}, cwd.empty() ? nullptr : cwd.c_str())) {- res.output = "failed to spawn process";- return res;- }-- std::atomic<bool> done{false};- std::atomic<bool> timed_out{false};-- std::thread timeout_thread([&]() {- auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_secs);- while (!done.load()) {- if (std::chrono::steady_clock::now() >= deadline) {- timed_out.store(true);- proc.terminate();- return;- }- std::this_thread::sleep_for(std::chrono::milliseconds(100));- }- });-- FILE * f = proc.stdout_file();- std::string output;- bool truncated = false;- if (f) {- char buf[4096];- while (fgets(buf, sizeof(buf), f) != nullptr) {- if (!truncated) {- size_t len = strlen(buf);- if (output.size() + len <= max_output) {- output.append(buf, len);- if (on_chunk && !on_chunk(console_output_to_utf8(std::string(buf, len)))) {- proc.terminate();- break;- }- } else {- size_t remaining = max_output - output.size();- output.append(buf, remaining);- if (on_chunk && remaining > 0) on_chunk(console_output_to_utf8(std::string(buf, remaining)));- truncated = true;- }- }- }- }-- done.store(true);- if (timeout_thread.joinable()) {- timeout_thread.join();- }-- res.exit_code = proc.join();-- res.output = console_output_to_utf8(output);- res.timed_out = timed_out.load();- if (truncated) {- res.output += "\n[output truncated]";- }- return res;+ return run_subprocess(args, max_output, timeout_secs, on_chunk, /*combine_stderr=*/true, cwd);}private:@@ -384,10 +408,8 @@ private:}static const std::unordered_set<std::string> & junk_dir_names() {- static const std::unordered_set<std::string> names = {- ".git", ".svn", ".hg", "node_modules", "__pycache__",- ".venv", "venv", "dist", "build", "target", ".cache", ".idea", ".vscode",- };+ static const std::unordered_set<std::string> names(+ std::begin(SERVER_TOOL_JUNK_DIR_NAMES), std::end(SERVER_TOOL_JUNK_DIR_NAMES));return names;}@@ -450,9 +472,274 @@ private:}};+// timeout for auxiliary isolate calls (stat/mkdir/ls/cp helpers); exec_shell_command uses its own+// caller-controlled timeout instead, enforced separately in run()+static constexpr int SERVER_TOOL_ISOLATE_EXEC_TIMEOUT = 15; // seconds+static constexpr size_t SERVER_TOOL_ISOLATE_READ_FILE_MAX_SIZE = 64 * 1024 * 1024; // 64 MB++// runs every tools_io operation as a command inside an isolate: a container, a remote host, ...+// the isolate is created, mounted, and torn down externally by the caller+// it must provide a POSIX environment: sh, cat, wc, mkdir, dirname, find, timeout+class tools_io_isolate : public tools_io {+public:+ // cwd, if non-empty, is used to resolve relative paths and as the working directory for run()+ explicit tools_io_isolate(std::string cwd = "") : cwd(std::move(cwd)) {}++ // resolves `path` against `cwd` if `path` is relative and `cwd` is set; otherwise returns `path` unchanged.+ // isolate paths are always POSIX-style ('/'), regardless of host OS.+ std::string resolve(const std::string & path) const override {+ if (cwd.empty() || (!path.empty() && path[0] == '/')) {+ return path;+ }+ return cwd + "/" + path;+ }++ bool is_directory(const std::string & path) const override {+ return shell_test("-d", resolve(path));+ }++ bool is_regular_file(const std::string & path) const override {+ return shell_test("-f", resolve(path));+ }++ bool file_size(const std::string & path, uintmax_t & out_size) const override {+ auto res = exec({"sh", "-c", "wc -c < \"$1\"", "_", resolve(path)}, 64, true);+ if (res.exit_code != 0 || res.timed_out) return false;+ try {+ size_t pos;+ out_size = (uintmax_t) std::stoull(res.output, &pos);+ } catch (...) {+ return false;+ }+ return true;+ }++ bool read_file(const std::string & path, std::string & out) const override {+ // combine_stderr=false: stderr must not be spliced into raw file bytes+ auto res = exec({"cat", "--", resolve(path)}, SERVER_TOOL_ISOLATE_READ_FILE_MAX_SIZE, false);+ if (res.exit_code != 0 || res.timed_out) return false;+ out = res.output;+ return true;+ }++ bool write_file(const std::string & path, const std::string & content) const override {+ std::string abs_path = resolve(path);++ std::error_code ec;+ fs::path tmp_dir = fs::temp_directory_path(ec);+ if (ec) return false;++ static std::atomic<uint64_t> tmp_counter{0};+ fs::path tmp = tmp_dir / string_format(+ "llama-tools-io-isolate-%zu-%llu.tmp",+ std::hash<std::thread::id>{}(std::this_thread::get_id()),+ (unsigned long long) tmp_counter.fetch_add(1));++ {+ std::ofstream f(tmp, std::ios::binary);+ if (!f) return false;+ f << content;+ if (!f) return false;+ }++ bool ok = shell_run({"sh", "-c", "mkdir -p \"$(dirname \"$1\")\"", "_", abs_path});+ if (ok) {+ ok = upload(tmp.string(), abs_path);+ }++ std::error_code rm_ec;+ fs::remove(tmp, rm_ec);+ return ok;+ }++ list_result list_entries(const std::string & base, int max_depth, list_kind kind) const override {+ list_result out;++ const std::string abs_base = resolve(base);+ if (!is_directory(base)) {+ out.err = "path does not exist or is not a directory";+ return out;+ }++ // git ls-files cannot list directories; use the walker when they are requested+ if (kind == list_kind::files) {+ auto res = exec(+ {"sh", "-c", "cd \"$1\" && git ls-files --cached --others --exclude-standard", "_", abs_base},+ SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, true);++ if (res.exit_code == 0 && !res.timed_out) {+ for (const auto & rel : split_lines(res.output, /*strip_dot_slash=*/false)) {+ if (max_depth > 0 && entry_depth(rel) > max_depth) continue;+ out.entries.push_back({rel, false});+ }+ return out;+ }+ }++ if (kind == list_kind::dirs || kind == list_kind::all) {+ for (auto & rel : find_entries(abs_base, max_depth, /*dirs=*/true, out.truncated)) {+ out.entries.push_back({std::move(rel), true});+ }+ }+ if (kind == list_kind::files || kind == list_kind::all) {+ for (auto & rel : find_entries(abs_base, max_depth, /*dirs=*/false, out.truncated)) {+ out.entries.push_back({std::move(rel), false});+ }+ }++ return out;+ }++ // wraps the command with an in-isolate `timeout`, since killing the host-side client+ // does not kill the process tree running inside the isolate+ exec_result run(+ const std::vector<std::string> & args,+ size_t max_output,+ int timeout_secs,+ const std::function<bool(const std::string &)> & on_chunk = nullptr) const override {+ std::vector<std::string> inner = {"timeout", std::to_string(timeout_secs) + "s"};+ inner.insert(inner.end(), args.begin(), args.end());+ // small buffer over timeout_secs so the in-isolate `timeout` has a chance to exit cleanly+ // before the host-side supervisory timeout forcibly kills the client+ return run_subprocess(+ build_argv(with_cwd(inner), /*needs_stdin=*/true),+ max_output, timeout_secs + 5, on_chunk, true);+ }++protected:+ // wrap `inner` (a complete POSIX argv) into the host-side argv that runs it in the isolate+ // a transport that re-parses its args in a remote shell (ssh) must join `inner` with shell_quote_join()+ virtual std::vector<std::string> build_argv(const std::vector<std::string> & inner, bool needs_stdin) const = 0;++ // copy a host file into the isolate, `isolate_path` is absolute and its parent already exists+ virtual bool upload(const std::string & host_path, const std::string & isolate_path) const = 0;++ // quote `argv` into a single string that a POSIX shell re-parses into exactly `argv`+ static std::string shell_quote_join(const std::vector<std::string> & argv) {+ std::string out;+ for (const auto & arg : argv) {+ if (!out.empty()) out += ' ';+ out += '\'';+ for (const char c : arg) {+ // a single quote cannot be escaped inside single quotes: close, escape, reopen+ if (c == '\'') out += "'\\''";+ else out += c;+ }+ out += '\'';+ }+ return out;+ }++private:+ std::string cwd;++ // set the working directory in the command itself, docker's `-w` has no equivalent on every transport+ // auxiliary calls do not need this, they use the absolute paths from resolve()+ std::vector<std::string> with_cwd(const std::vector<std::string> & inner) const {+ if (cwd.empty()) {+ return inner;+ }+ // 127 is what a shell reports for a command it could not run+ std::vector<std::string> out = {"sh", "-c", "cd \"$1\" || exit 127; shift; exec \"$@\"", "_", cwd};+ out.insert(out.end(), inner.begin(), inner.end());+ return out;+ }++ exec_result exec(const std::vector<std::string> & inner, size_t max_output, bool combine_stderr) const {+ return run_subprocess(+ build_argv(inner, /*needs_stdin=*/false),+ max_output, SERVER_TOOL_ISOLATE_EXEC_TIMEOUT, nullptr, combine_stderr);+ }++ bool shell_run(const std::vector<std::string> & inner) const {+ auto res = exec(inner, 4096, true);+ return res.exit_code == 0 && !res.timed_out;+ }++ bool shell_test(const char * flag, const std::string & path) const {+ return shell_run({"sh", "-c", std::string("[ ") + flag + " \"$1\" ]", "_", path});+ }++ static std::vector<std::string> split_lines(const std::string & text, bool strip_dot_slash) {+ std::vector<std::string> result;+ std::istringstream iss(text);+ std::string line;+ while (std::getline(iss, line)) {+ if (!line.empty() && line.back() == '\r') line.pop_back();+ if (line.empty()) continue;+ if (strip_dot_slash && line.rfind("./", 0) == 0) line = line.substr(2);+ std::replace(line.begin(), line.end(), '\\', '/');+ result.push_back(line);+ }+ return result;+ }++ // one `find` pass in the isolate. junk directories stay selectable but are never descended into,+ // and -mindepth/-maxdepth keep a busybox image working as well as a GNU one+ std::vector<std::string> find_entries(const std::string & abs_base, int max_depth, bool dirs, bool & truncated) const {+ std::string prune_expr;+ for (const char * n : SERVER_TOOL_JUNK_DIR_NAMES) {+ if (!prune_expr.empty()) prune_expr += " -o ";+ prune_expr += std::string("-name ") + n;+ }++ std::string cmd = "cd \"$1\" && find . -mindepth 1";+ if (max_depth > 0) {+ cmd += " -maxdepth " + std::to_string(max_depth);+ }+ cmd += " \\( " + prune_expr + " \\) -prune";+ cmd += dirs ? " -print -o -type d -print" : " -o -type f -print";++ auto res = exec({"sh", "-c", cmd, "_", abs_base}, SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, true);+ truncated = truncated || res.timed_out;+ return split_lines(res.output, /*strip_dot_slash=*/true);+ }+};++// an already-running docker container, driven through `docker exec` and `docker cp`+class tools_io_docker : public tools_io_isolate {+public:+ tools_io_docker(std::string container_id, std::string cwd = "")+ : tools_io_isolate(std::move(cwd)), container_id(std::move(container_id)) {}++protected:+ std::vector<std::string> build_argv(const std::vector<std::string> & inner, bool needs_stdin) const override {+ std::vector<std::string> argv = {"docker", "exec"};+ if (needs_stdin) {+ argv.push_back("-i");+ }+ argv.push_back(container_id);+ argv.insert(argv.end(), inner.begin(), inner.end());+ return argv;+ }++ bool upload(const std::string & host_path, const std::string & isolate_path) const override {+ auto res = run_subprocess(+ {"docker", "cp", host_path, container_id + ":" + isolate_path},+ 4096, SERVER_TOOL_ISOLATE_EXEC_TIMEOUT, nullptr, true);+ return res.exit_code == 0 && !res.timed_out;+ }++private:+ std::string container_id;+};++// runtime spec used by --tools-runtime and the x-tool-runtime header+// this is the only scheme for now, ssh: and podman: can be added next to it+static const std::string SERVER_TOOL_RUNTIME_DOCKER_CONTAINER = "docker-container:";++// an empty runtime runs the tools on the hoststatic std::unique_ptr<tools_io> make_tools_io(const json & params) {- std::string cwd = json_value(params, "cwd", std::string());- return std::make_unique<tools_io_basic>(cwd);+ std::string cwd = json_value(params, "cwd", std::string());+ std::string runtime = json_value(params, "runtime", std::string());+ if (runtime.empty()) {+ return std::make_unique<tools_io_basic>(cwd);+ }+ if (runtime.rfind(SERVER_TOOL_RUNTIME_DOCKER_CONTAINER, 0) == 0) {+ return std::make_unique<tools_io_docker>(runtime.substr(SERVER_TOOL_RUNTIME_DOCKER_CONTAINER.size()), cwd);+ }+ // do not fall back to the host, the caller asked for an isolate+ throw std::runtime_error("unknown tool runtime: " + runtime);}// no '/' in pattern -> match basename at any depth; else match full relative path@@ -861,8 +1148,11 @@ struct server_tool_exec_shell_command : server_tool {timeout = std::min(timeout, SERVER_TOOL_EXEC_SHELL_COMMAND_MAX_TIMEOUT);max_output = std::min(max_output, SERVER_TOOL_EXEC_SHELL_COMMAND_MAX_OUTPUT_SIZE);+ // an isolate is always POSIX regardless of host OS, so it always gets `sh -c`#ifdef _WIN32- std::vector<std::string> args = {"cmd", "/c", command};+ std::vector<std::string> args = !json_value(params, "runtime", std::string()).empty()+ ? std::vector<std::string>{"sh", "-c", command}+ : std::vector<std::string>{"cmd", "/c", command};#elsestd::vector<std::string> args = {"sh", "-c", command};#endif@@ -1355,11 +1645,16 @@ struct server_tool_get_info : server_tool {json invoke(json params, server_tool::stream *) const override {auto io = make_tools_io(params);+ // inside an isolate, we always use the linux command#ifdef _WIN32- auto res = io->run({"cmd", "/c", "ver"}, SERVER_TOOL_GET_INFO_MAX_OUTPUT, SERVER_TOOL_GET_INFO_TIMEOUT);+ std::vector<std::string> args = !json_value(params, "runtime", std::string()).empty()+ ? std::vector<std::string>{"uname", "-a"}+ : std::vector<std::string>{"cmd", "/c", "ver"};#else- auto res = io->run({"uname", "-a"}, SERVER_TOOL_GET_INFO_MAX_OUTPUT, SERVER_TOOL_GET_INFO_TIMEOUT);+ std::vector<std::string> args = {"uname", "-a"};#endif++ auto res = io->run(args, SERVER_TOOL_GET_INFO_MAX_OUTPUT, SERVER_TOOL_GET_INFO_TIMEOUT);// "ver" prints a blank line before the version, so the output is stripped on both ends;// a failed spawn or a timeout leaves a diagnostic in res.output, which is not an OS namestd::string os_info = res.exit_code == 0 && !res.timed_out ? string_strip(res.output) : "unknown";@@ -1461,6 +1756,103 @@ struct server_mcp_tool : server_tool {}};+// owns the docker container used as the sandboxed runtime for tool invocations, as configured by+// --tools-runtime. "spawned" mode starts and stops the container itself; "existing" mode just reuses+// a container id the user already has running and never stops it.+struct server_tools_docker_runtime {+ server_tools_docker_runtime(const server_tools_docker_runtime &) = delete;++ explicit server_tools_docker_runtime(const std::string & spec) {+ static const std::string docker_prefix = "docker:";+ if (spec.rfind(docker_prefix, 0) == 0) {+ spawned = true;+ image = spec.substr(docker_prefix.size());+ if (image.empty()) {+ throw std::runtime_error("--tools-runtime docker:<image> requires an image name");+ }+ spawn();+ } else if (spec.rfind(SERVER_TOOL_RUNTIME_DOCKER_CONTAINER, 0) == 0) {+ spawned = false;+ container_id = spec.substr(SERVER_TOOL_RUNTIME_DOCKER_CONTAINER.size());+ if (container_id.empty()) {+ throw std::runtime_error("--tools-runtime docker-container:<id> requires a container id");+ }+ } else {+ throw std::runtime_error("unknown --tools-runtime option: " + spec);+ }+ }++ ~server_tools_docker_runtime() {+ if (spawned && !container_id.empty()) {+ // closing stdin signals the container's shell (its pid 1) to exit; --rm then removes it+ proc.close_stdin();+ proc.join();+ }+ }++ // container id to use for the next tool call; respawns a spawned container that died on its own,+ // or throws if an externally-managed one is no longer reachable+ std::string get_container_id() {+ std::lock_guard<std::mutex> lock(mutex);+ if (!spawned) {+ if (!is_running(container_id)) {+ throw std::runtime_error(string_format(+ "docker container \"%s\" is no longer running, restart it to keep using tools",+ container_id.c_str()));+ }+ return container_id;+ }++ if (!proc.alive()) {+ SRV_WRN("docker tools runtime container \"%s\" died, respawning\n", container_id.c_str());+ spawn();+ }+ return container_id;+ }++private:+ bool spawned = false;+ std::string image; // spawned mode only+ std::string container_id;+ common_subproc proc; // spawned mode only: `docker run` client that keeps the container alive+ std::mutex mutex;++ // spawns "docker run --rm -i <image> sh" and keeps its stdin open; the shell blocks reading stdin,+ // so the container stays alive until we close it (see destructor) or it is killed from the outside+ void spawn() {+ std::error_code ec;+ fs::path cidfile = fs::temp_directory_path(ec) / string_format(+ "llama-tools-runtime-cid-%zu.tmp", std::hash<std::thread::id>{}(std::this_thread::get_id()));+ fs::remove(cidfile, ec);++ std::vector<std::string> args = {"docker", "run", "--rm", "-i", "--cidfile", cidfile.string(), image, "sh"};+ int options = subprocess_option_no_window+ | subprocess_option_inherit_environment+ | subprocess_option_search_user_path;+ if (!proc.create(args, options)) {+ throw std::runtime_error("failed to spawn docker container for tools runtime (image: " + image + ")");+ }++ std::string cid;+ for (int i = 0; i < 100 && cid.empty(); i++) {+ std::ifstream f(cidfile);+ if (f) std::getline(f, cid);+ if (cid.empty()) std::this_thread::sleep_for(std::chrono::milliseconds(100));+ }+ fs::remove(cidfile, ec);+ if (cid.empty()) {+ proc.terminate();+ throw std::runtime_error("timed out waiting for docker container to start (image: " + image + ")");+ }+ container_id = cid;+ }++ static bool is_running(const std::string & id) {+ auto res = run_subprocess({"docker", "inspect", "-f", "{{.State.Running}}", id}, 16, 5, nullptr, true);+ return res.exit_code == 0 && !res.timed_out && res.output.rfind("true", 0) == 0;+ }+};+static server_tool & find_tool(std::vector<std::unique_ptr<server_tool>> & tools, const std::string & name, bool require_stream) {for (auto & t : tools) {if (t->name == name) {@@ -1506,8 +1898,16 @@ static std::string get_header(const std::map<std::string, std::string> & headersreturn default_value;}+server_tools::server_tools() = default;+server_tools::~server_tools() = default;+void server_tools::setup(const std::vector<std::string> & enabled_tools,- server_mcp & mcp_mgr) {+ server_mcp & mcp_mgr,+ const std::string & tools_runtime) {+ if (!tools_runtime.empty()) {+ docker_runtime = std::make_unique<server_tools_docker_runtime>(tools_runtime);+ }+if (!enabled_tools.empty()) {if (!common_subproc::is_supported()) {throw std::runtime_error("subprocess is not enabled on this build");@@ -1590,11 +1990,26 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools,bool stream = body.value("stream", false);// accept x-tool-cwd header to override of the process+ if (params.contains("cwd")) {+ params.erase("cwd");+ }auto cwd = get_header(req.headers, "x-tool-cwd");if (!cwd.empty()) {params["cwd"] = cwd;}+ // accept x-tool-runtime header to route tool I/O through an isolate, e.g. "docker-container:<id>";+ // falls back to the --tools-runtime isolate, if configured+ if (params.contains("runtime")) {+ params.erase("runtime");+ }+ auto runtime = get_header(req.headers, "x-tool-runtime");+ if (!runtime.empty()) {+ params["runtime"] = runtime;+ } else if (docker_runtime) {+ params["runtime"] = SERVER_TOOL_RUNTIME_DOCKER_CONTAINER + docker_runtime->get_container_id();+ }+server_tool & tool = find_tool(tools, tool_name, stream);if (stream) {diff --git a/tools/server/server-tools.h b/tools/server/server-tools.hindex 601399ee93..7f70e6767e 100644--- a/tools/server/server-tools.h+++ b/tools/server/server-tools.h@@ -30,6 +30,8 @@ struct server_tool {json to_json() const;};+struct server_tools_docker_runtime; // impl detail, defined in server-tools.cpp+struct server_tools {std::vector<std::unique_ptr<server_tool>> tools;@@ -37,9 +39,16 @@ struct server_tools {server_response queue_res;std::atomic<int> res_id{0};+ // set when --tools-runtime is configured; owns the docker container used to run tools, if any+ std::unique_ptr<server_tools_docker_runtime> docker_runtime;+void setup(const std::vector<std::string> & enabled_tools,- server_mcp & mcp_mgr);+ server_mcp & mcp_mgr,+ const std::string & tools_runtime);server_http_context::handler_t handle_get;server_http_context::handler_t handle_post;++ server_tools();+ ~server_tools();};diff --git a/tools/server/server.cpp b/tools/server/server.cppindex aafb1f3079..1b2e6edb4e 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -338,7 +338,7 @@ int llama_server(common_params & params, int argc, char ** argv) {if (!params.server_tools.empty() || !mcp_mgr.empty()) {try {- tools.setup(params.server_tools, mcp_mgr);+ tools.setup(params.server_tools, mcp_mgr, params.server_tools_runtime);} catch (const std::exception & e) {SRV_ERR("tools setup failed: %s\n", e.what());return 1;@@ -348,6 +348,9 @@ int llama_server(common_params & params, int argc, char ** argv) {if (!params.server_tools.empty()) {warn_names.push_back("built-in tools (experimental)");}+ if (!params.server_tools_runtime.empty()) {+ warn_names.push_back("tools runtime (experimental)");+ }if (!mcp_mgr.empty()) {warn_names.push_back("MCP servers (experimental)");}diff --git a/tools/server/tests/unit/test_tools_builtin.py b/tools/server/tests/unit/test_tools_builtin.pyindex 11c82e690a..c651e8e72d 100755--- a/tools/server/tests/unit/test_tools_builtin.py+++ b/tools/server/tests/unit/test_tools_builtin.py@@ -1,4 +1,6 @@import os+import shutil+import subprocessimport pytestfrom utils import *@@ -146,6 +148,95 @@ def test_tools_builtin_cwd_header():os.remove(marker_path)+def _docker_unavailable_reason() -> str | None:+ """None if docker can be used to run a container, otherwise the reason it can't."""+ docker_bin = shutil.which("docker")+ if docker_bin is None:+ return "docker is not installed"+ try:+ subprocess.run([docker_bin, "info"], capture_output=True, timeout=5, check=True)+ except Exception as e:+ return f"docker daemon is not usable: {e}"+ return None+++def docker_container():+ reason = _docker_unavailable_reason()+ if reason is not None:+ pytest.skip(reason) # ty: ignore[too-many-positional-arguments, invalid-argument-type]++ proc = subprocess.run(+ ["docker", "run", "-d", "--rm", "busybox", "sleep", "300"],+ capture_output=True, text=True,+ )+ if proc.returncode != 0:+ pytest.skip(f"failed to start docker container: {proc.stderr.strip()}") # ty: ignore[too-many-positional-arguments, invalid-argument-type]++ container_id = proc.stdout.strip()+ try:+ yield container_id+ finally:+ subprocess.run(["docker", "rm", "-f", container_id], capture_output=True)+++def test_tools_builtin_runtime_header(docker_container: str):+ global server+ server.start()++ headers = {"x-tool-runtime": f"docker-container:{docker_container}", "x-tool-cwd": "/tmp"}++ write_res = call_tool("write_file", {"path": "test.log", "content": "hello docker\n"}, headers=headers)+ assert write_res["result"] == "file written successfully"++ read_res = call_tool("read_file", {"path": "test.log"}, headers=headers)+ assert read_res["plain_text_response"] == "hello docker\n"++ exec_res = call_tool("exec_shell_command", {"command": "cat test.log"}, headers=headers)+ assert "hello docker" in exec_res["plain_text_response"]+++def test_tools_builtin_runtime_header_unknown_scheme():+ global server+ server.start()++ # an unknown runtime must fail, never silently fall back to running on the host+ res = server.make_request("POST", "/tools",+ data={"tool": "exec_shell_command", "params": {"command": "echo hi"}},+ headers={"x-tool-runtime": "ssh:example.com"})+ assert res.status_code == 500, res.body+ assert "unknown tool runtime" in str(res.body)+++def test_tools_builtin_docker_runtime_cleans_up_spawned_container():+ reason = _docker_unavailable_reason()+ if reason is not None:+ pytest.skip(reason) # ty: ignore[too-many-positional-arguments, invalid-argument-type]++ global server+ server.server_tools_runtime = "docker:busybox"+ server.start()++ # exec_shell_command runs inside the container spawned for --tools-runtime; docker sets+ # the container's hostname to its own short id, so this also tells us which one to check+ res = call_tool("exec_shell_command", {"command": "hostname"})+ container_id = res["plain_text_response"].splitlines()[0].strip()+ assert len(container_id) >= 8, res++ running = subprocess.run(+ ["docker", "inspect", "-f", "{{.State.Running}}", container_id],+ capture_output=True, text=True,+ )+ assert running.returncode == 0 and running.stdout.strip() == "true", running.stderr++ server.stop()++ # a clean server shutdown must stop and remove the container it spawned (it runs with --rm),+ # not leave it behind as an abandoned child+ leftover = subprocess.run(["docker", "inspect", container_id], capture_output=True, text=True)+ assert leftover.returncode != 0, f"container {container_id} was not cleaned up after server exit"++def test_tools_builtin_edit_file_rejects_overlapping_edits():global serverserver.start()diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex 3416f0bfde..fffe07a671 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -115,6 +115,7 @@ class ServerProcess:backend_sampling: bool = Falsegcp_compat: bool = Falseserver_tools: str | None = None+ server_tools_runtime: str | None = Nonemcp_servers_config: str | None = Nonemcp_servers_json: str | None = Nonecors_origins: str | None = None@@ -270,6 +271,8 @@ class ServerProcess:server_args.append("--ui-mcp-proxy")if self.server_tools:server_args.extend(["--tools", self.server_tools])+ if self.server_tools_runtime:+ server_args.extend(["--tools-runtime", self.server_tools_runtime])if self.mcp_servers_config:server_args.extend(["--mcp-servers-config", self.mcp_servers_config])if self.mcp_servers_json: