LLM inference in C/C++
C++ ggml
common: add system-level config file (#26118)
Johnathan Craig Maudlin8e7f22b67ef4
diff --git a/common/arg.cpp b/common/arg.cppindex b2fddfe5f2..9fe7d66e24 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -35,6 +35,7 @@#include <regex>#include <set>#include <string>+#include <system_error>#include <thread> // for hardware_concurrency#include <vector>@@ -704,12 +705,61 @@ void common_models_handler_apply(common_models_handler & handler, common_params// CLI argument parsing functions//+// apply config files (if present), a later file overrides an earlier one:+// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)+// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)+static void common_params_apply_system_config(common_params & params, llama_example ex) {+ std::vector<std::string> paths;++#if defined(_WIN32)+ const std::string program_data = common_get_env("PROGRAMDATA");+ if (!program_data.empty()) {+ paths.push_back(program_data + "\\llama.cpp\\config.ini");+ }+#else+ paths.push_back("/etc/llama.cpp/config.ini");+#endif++ try {+ paths.push_back(fs_get_config_directory() + "config.ini");+ } catch (const std::exception & e) {+ LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());+ }++ std::vector<std::string> found;+ for (const auto & path : paths) {+ std::error_code ec;+ if (std::filesystem::exists(path, ec)) {+ found.push_back(path);+ }+ }+ if (found.empty()) {+ return;+ }++ common_preset_context ctx(ex);+ ctx.ignore_unknown_keys = true; // the same config file is shared by all programs+ for (const auto & path : found) {+ LOG_INF("using config file: %s\n", path.c_str());+ common_preset global;+ common_presets presets = ctx.load_from_ini(path, global);+ global.apply_to_params(params);+ auto it = presets.find(COMMON_PRESET_DEFAULT_NAME);+ if (it != presets.end()) {+ it->second.apply_to_params(params);+ }+ }+}+static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {common_params & params = ctx_arg.params;// setup log directly from params.verbosity: see tools/cli/cli.cppcommon_log_set_verbosity_thold(params.verbosity);+ // config file applies first, so env variables and CLI arguments override it+ common_params_apply_system_config(params, ctx_arg.ex);+std::unordered_map<std::string, std::pair<common_arg *, bool>> arg_to_options;for (auto & opt : ctx_arg.options) {for (const auto & arg : opt.args) {diff --git a/common/common.cpp b/common/common.cppindex 2e3f14cd1c..5639bdf412 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1019,20 +1019,21 @@ std::string fs_get_cache_directory() {std::string cache_directory = "";auto ensure_trailing_slash = [](std::string p) {// Make sure to add trailing slash- if (p.back() != DIRECTORY_SEPARATOR) {+ if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {p += DIRECTORY_SEPARATOR;}return p;};- if (getenv("LLAMA_CACHE")) {- cache_directory = std::getenv("LLAMA_CACHE");- } else {+ cache_directory = common_get_env("LLAMA_CACHE");+ if (cache_directory.empty()) {#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \defined(__OpenBSD__) || defined(__NetBSD__)- if (std::getenv("XDG_CACHE_HOME")) {- cache_directory = std::getenv("XDG_CACHE_HOME");- } else if (std::getenv("HOME")) {- cache_directory = std::getenv("HOME") + std::string("/.cache/");+ const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");+ const std::string home = common_get_env("HOME");+ if (!xdg_cache_home.empty()) {+ cache_directory = xdg_cache_home;+ } else if (!home.empty()) {+ cache_directory = home + "/.cache/";} else {#if defined(__linux__)/* no $HOME is defined, fallback to getpwuid */@@ -1047,9 +1048,16 @@ std::string fs_get_cache_directory() {#endif /* defined(__linux__) */}#elif defined(__APPLE__)- cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");+ cache_directory = common_get_env("HOME");+ if (cache_directory.empty()) {+ throw std::runtime_error("Failed to find $HOME directory");+ }+ cache_directory += "/Library/Caches/";#elif defined(_WIN32)- cache_directory = std::getenv("LOCALAPPDATA");+ cache_directory = common_get_env("LOCALAPPDATA");+ if (cache_directory.empty()) {+ throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");+ }#elif defined(__EMSCRIPTEN__)GGML_ABORT("not implemented on this platform");#else@@ -1061,6 +1069,51 @@ std::string fs_get_cache_directory() {return ensure_trailing_slash(cache_directory);}+std::string fs_get_config_directory() {+ std::string config_directory = "";+ auto ensure_trailing_slash = [](std::string p) {+ if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {+ p += DIRECTORY_SEPARATOR;+ }+ return p;+ };+#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \+ defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)+ const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");+ const std::string home = common_get_env("HOME");+ if (!xdg_config_home.empty()) {+ config_directory = xdg_config_home;+ } else if (!home.empty()) {+ config_directory = home + "/.config/";+ } else {+#if defined(__linux__)+ /* no $HOME is defined, fallback to getpwuid */+ struct passwd *pw = getpwuid(getuid());+ if ((!pw) || (!pw->pw_dir)) {+ throw std::runtime_error("Failed to find $HOME directory");+ }++ config_directory = std::string(pw->pw_dir) + std::string("/.config/");+#else+ throw std::runtime_error("Failed to find $HOME directory");+#endif+ }+#elif defined(_WIN32)+ config_directory = common_get_env("APPDATA");+ if (config_directory.empty()) {+ throw std::runtime_error("Failed to find %APPDATA% directory");+ }+#elif defined(__EMSCRIPTEN__)+ // caller decides what to do when there is no config directory+ throw std::runtime_error("not implemented on this platform");+#else+# error Unknown architecture+#endif+ config_directory = ensure_trailing_slash(config_directory);+ config_directory += "llama.cpp";+ return ensure_trailing_slash(config_directory);+}+std::string fs_get_cache_file(const std::string & filename) {GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);std::string cache_directory = fs_get_cache_directory();diff --git a/common/common.h b/common/common.hindex d485d4fb41..b16bd3bb68 100644--- a/common/common.h+++ b/common/common.h@@ -881,6 +881,7 @@ bool fs_is_directory(const std::string & path);std::string fs_get_cache_directory();std::string fs_get_cache_file(const std::string & filename);+std::string fs_get_config_directory();struct common_file_info {std::string path;diff --git a/common/preset.cpp b/common/preset.cppindex eb0c60b09c..0b29af8834 100644--- a/common/preset.cpp+++ b/common/preset.cpp@@ -322,6 +322,8 @@ common_presets common_preset_context::load_from_ini(const std::string & path, copreset.options[opt] = value;}LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());+ } else if (ignore_unknown_keys) {+ LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.c_str());} else {throw std::runtime_error(string_format("option '%s' not recognized in preset '%s'",diff --git a/common/preset.h b/common/preset.hindex 52935ebde8..d8fc3915bc 100644--- a/common/preset.h+++ b/common/preset.h@@ -59,6 +59,10 @@ struct common_preset_context {bool filter_allowed_keys = false;std::set<std::string> allowed_keys;+ // if true, options unknown to the current example are skipped instead of being an error+ // used for config files shared by all binaries, where each binary only knows a subset of options+ bool ignore_unknown_keys = false;+// if only_remote_allowed is true, only accept whitelisted keyscommon_preset_context(llama_example ex);diff --git a/docs/preset.md b/docs/preset.mdindex 85762a420b..3d85467e87 100644--- a/docs/preset.md+++ b/docs/preset.md@@ -4,7 +4,7 @@The INI preset feature, introduced in [PR#17859](https://github.com/ggml-org/llama.cpp/pull/17859), allows users to create reusable and shareable parameter configurations for llama.cpp.-### Using Presets with the Server+## Using Presets with the ServerWhen running multiple models on the server (router mode), INI preset files can be used to configure model-specific parameters. Please refer to the [server documentation](../tools/server/README.md) for more details.@@ -93,3 +93,18 @@ llama-server -hf user/repo:gpt-oss-120b-hf```Please make sure to provide the correct `hf-repo` for each child preset. Otherwise, you may get error: `The specified tag is not a valid quantization scheme.`++## System-level config++The system-level config, added in PR [#26118](https://github.com/ggml-org/llama.cpp/pull/26118), allows sharing the same set of options among multiple tools and examples. Unlike the sections above, it is not limited to the server.++These files are loaded on startup if present. A later file overrides an earlier one:+1. System-wide: `/etc/llama.cpp/config.ini` (or `%PROGRAMDATA%\llama.cpp\config.ini` on Windows)+2. User-level: `$XDG_CONFIG_HOME/llama.cpp/config.ini`, `~/.config/llama.cpp/config.ini` by default (or `%APPDATA%\llama.cpp\config.ini` on Windows)++The config file is applied first, then its options are overridden by ENV variables, CLI arguments and model presets (in router mode).++Note:+- Only the `[*]` and default sections are used; options written before any section header belong to "default. Named sections are ignored+- Tool-specific options can be specified, but will be ignored (with a warning) if the example doesn't support it<br/>Example: if you specify `port = 1234`, only `llama-server` will use it, other examples will ignore it+- `model` or `hf-repo` are not recommended to be configured system-level, because it may introduce conflicts<br/>Example: a `hf-repo` in the config file still takes effect when you pass `-m` on the command line, so you may load a different model than expected