LLM inference in C/C++
C++ ggml
server: add --connect, allow p2p access to server outside LAN (#28277)
Xuan-Son Nguyen7adf160c411e
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 0a732a9146..6f828b94be 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -35,7 +35,7 @@ permissions:env:GH_TOKEN: ${{ github.token }}BRANCH_NAME: ${{ github.head_ref || github.ref_name }}- CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"+ CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DLLAMA_CONNECT=ON"# note: run this workflow one at a time for better cache reuseconcurrency:@@ -1478,6 +1478,7 @@ jobs:-DCMAKE_INSTALL_RPATH='$ORIGIN' \-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \-DGGML_NATIVE=OFF \+ -DLLAMA_CONNECT=ON \-DGGML_SYCL_F16=${{ matrix.fp16 }}time cmake --build build --config Release -j $(nproc)diff --git a/CMakeLists.txt b/CMakeLists.txtindex 12f45558f5..d091a1411b 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -136,6 +136,7 @@ option(LLAMA_BUILD_SERVER "llama: build server example"option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)+option(LLAMA_CONNECT "llama: include the llama-connect tool (downloads a prebuilt binary)" OFF)option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT})option(LLAMA_TESTS_INSTALL "llama: install tests" ON)diff --git a/common/arg.cpp b/common/arg.cppindex 8b302d5d77..267b848749 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -3494,6 +3494,26 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.ui = value;}).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI"));+ add_opt(common_arg(+ {"--connect"},+ string_format("(experimental) open a peer-to-peer tunnel for server using llama-connect (default: %s)", params.server_connect ? "enabled" : "disabled"),+ [](common_params & params) {+ params.server_connect = true;+ }+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CONNECT"));+ add_opt(common_arg(+ {"--connect-code"}, "CODE",+ "manually specify a 40-character code for --connect (default: generate a new one for each run)",+ [](common_params & params, const std::string & value) {+ std::string val = value;+ string_replace_all(val, "-", "");+ string_replace_all(val, " ", "");+ if (val.size() != 40) {+ throw std::invalid_argument(string_format("error: invalid connect code '%s', must be 40 characters\n", value.c_str()));+ }+ params.server_connect_code = val;+ }+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CONNECT_CODE"));add_opt(common_arg({"--embedding", "--embeddings"},string_format("restrict to only support embedding use case; use only with dedicated embedding models (default: %s)", params.embedding ? "enabled" : "disabled"),diff --git a/common/common.h b/common/common.hindex 0c2d5e46f1..b172b4efe7 100644--- a/common/common.h+++ b/common/common.h@@ -642,6 +642,10 @@ struct common_params {int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise+ // llama-connect params+ bool server_connect = false;+ std::string server_connect_code = "";+std::string public_path = ""; // NOLINTstd::string api_prefix = ""; // NOLINTstd::string chat_template = ""; // NOLINTdiff --git a/scripts/connect-download.cmake b/scripts/connect-download.cmakenew file mode 100644index 0000000000..da340b2688--- /dev/null+++ b/scripts/connect-download.cmake@@ -0,0 +1,86 @@+# download the prebuilt llama-connect binary for the given platform+# the release assets are made by: https://github.com/ggml-org/llama-connect/blob/master/.github/workflows/build.yml++cmake_minimum_required(VERSION 3.18)++set(REPO "" CACHE STRING "GitHub repository to download from (owner/name)")+set(VERSION "" CACHE STRING "Release tag to download, or 'latest'")+set(PLATFORM "" CACHE STRING "Platform suffix of the release asset (ex: linux-x64)")+set(OUT_DIR "" CACHE STRING "Directory to place the binary into")+set(WORK_DIR "" CACHE STRING "Scratch directory used for the download")++if (PLATFORM MATCHES "^win-")+ set(ASSET "llama-connect-${PLATFORM}.zip")+ set(BINARY "llama-connect.exe")+else()+ set(ASSET "llama-connect-${PLATFORM}.tar.gz")+ set(BINARY "llama-connect")+endif()++if (VERSION STREQUAL "latest")+ set(URL "https://github.com/${REPO}/releases/latest/download/${ASSET}")+else()+ set(URL "https://github.com/${REPO}/releases/download/${VERSION}/${ASSET}")+endif()++set(DST "${OUT_DIR}/${BINARY}")+set(ARCHIVE "${WORK_DIR}/${ASSET}")+set(STAMP "${WORK_DIR}/${BINARY}.url")++# skip if already downloaded from the same URL+if (EXISTS "${DST}" AND EXISTS "${STAMP}")+ file(READ "${STAMP}" prev_url)+ if (prev_url STREQUAL "${URL}")+ return()+ endif()+endif()++file(REMOVE "${STAMP}")+file(MAKE_DIRECTORY "${WORK_DIR}")++message(STATUS "llama-connect: downloading ${URL}")++# retry a few times, this runs in CI where a transient failure must not break the build+foreach (attempt RANGE 1 3)+ file(DOWNLOAD "${URL}" "${ARCHIVE}" STATUS status TLS_VERIFY ON INACTIVITY_TIMEOUT 60)++ list(GET status 0 code)+ list(GET status 1 msg)++ if (code EQUAL 0)+ break()+ endif()++ message(STATUS "llama-connect: download attempt ${attempt} failed (${code}: ${msg})")+ file(REMOVE "${ARCHIVE}")+endforeach()++if (NOT code EQUAL 0)+ file(REMOVE "${ARCHIVE}")+ message(FATAL_ERROR+ "llama-connect: failed to download ${URL} (${code}: ${msg})\n"+ " use -DLLAMA_CONNECT_VERSION=<tag> to select another release, "+ "or -DLLAMA_CONNECT=OFF to skip this tool")+endif()++set(EXTRACT_DIR "${WORK_DIR}/extract")++file(REMOVE_RECURSE "${EXTRACT_DIR}")+file(MAKE_DIRECTORY "${EXTRACT_DIR}")+file(ARCHIVE_EXTRACT INPUT "${ARCHIVE}" DESTINATION "${EXTRACT_DIR}")++if (NOT EXISTS "${EXTRACT_DIR}/${BINARY}")+ message(FATAL_ERROR "llama-connect: ${ASSET} does not contain ${BINARY}")+endif()++file(MAKE_DIRECTORY "${OUT_DIR}")+file(COPY "${EXTRACT_DIR}/${BINARY}" DESTINATION "${OUT_DIR}"+ FILE_PERMISSIONS+ OWNER_READ OWNER_WRITE OWNER_EXECUTE+ GROUP_READ GROUP_EXECUTE+ WORLD_READ WORLD_EXECUTE+)++file(WRITE "${STAMP}" "${URL}")++message(STATUS "llama-connect: ready at ${DST}")diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txtindex c8ad1db436..6a6592c7bf 100644--- a/tools/CMakeLists.txt+++ b/tools/CMakeLists.txt@@ -32,6 +32,9 @@ else()if (GGML_RPC)add_subdirectory(rpc)endif()+ if (LLAMA_CONNECT)+ add_subdirectory(connect)+ endif()if (NOT GGML_BACKEND_DL AND GGML_CPU)# these tools use backends directly (no dynamic loading) and depend on CPU backend symbolsadd_subdirectory(cvector-generator)diff --git a/tools/cli/README.md b/tools/cli/README.mdindex be8fb09d23..f1f6e6ed42 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -177,7 +177,7 @@| `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,<br/>or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)<br/>(env: LLAMA_ARG_REASONING_EFFORT) || `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) || `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)<br/>(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) |-| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) |+| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) || `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) || `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) || `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)<br/>(env: LLAMA_ARG_SKIP_CHAT_PARSING) |diff --git a/tools/completion/README.md b/tools/completion/README.mdindex c396609abf..7174a04ff9 100644--- a/tools/completion/README.md+++ b/tools/completion/README.md@@ -255,7 +255,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1| `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,<br/>or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)<br/>(env: LLAMA_ARG_REASONING_EFFORT) || `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) || `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)<br/>(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) |-| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) |+| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) || `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) || `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) || `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)<br/>(env: LLAMA_ARG_SKIP_CHAT_PARSING) |diff --git a/tools/connect/CMakeLists.txt b/tools/connect/CMakeLists.txtnew file mode 100644index 0000000000..9a21ea7a2b--- /dev/null+++ b/tools/connect/CMakeLists.txt@@ -0,0 +1,85 @@+# llama-connect is developed in a separate repo, nothing is compiled here+# we only download the prebuilt binary, see scripts/connect-download.cmake++set(TARGET llama-connect)++set(LLAMA_CONNECT_REPO "ggml-org/llama-connect" CACHE STRING "llama-connect: GitHub repository to download from")+set(LLAMA_CONNECT_VERSION "latest" CACHE STRING "llama-connect: release tag to download (ex: v0.0.1), or 'latest'")++# map the target platform to the release asset name+set(LLAMA_CONNECT_PLATFORM "")++if (APPLE)+ set(_arch "${CMAKE_SYSTEM_PROCESSOR}")+ if (CMAKE_OSX_ARCHITECTURES)+ list(LENGTH CMAKE_OSX_ARCHITECTURES _narch)+ if (_narch EQUAL 1)+ set(_arch "${CMAKE_OSX_ARCHITECTURES}")+ else()+ set(_arch "universal") # no such asset, so the tool is skipped below+ endif()+ endif()++ # note: there is no macos-x64 asset, it is not built upstream (no Intel macOS runners)+ if (_arch MATCHES "^(arm64|aarch64)$")+ set(LLAMA_CONNECT_PLATFORM "macos-arm64")+ endif()+elseif (WIN32)+ # with Visual Studio generators, the target arch comes from -A+ set(_arch "${CMAKE_SYSTEM_PROCESSOR}")+ if (CMAKE_GENERATOR_PLATFORM)+ set(_arch "${CMAKE_GENERATOR_PLATFORM}")+ endif()++ if (_arch MATCHES "^(x86_64|AMD64|amd64|x64)$")+ set(LLAMA_CONNECT_PLATFORM "win-x64")+ endif()+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux")+ if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|x64)$")+ set(LLAMA_CONNECT_PLATFORM "linux-x64")+ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$")+ set(LLAMA_CONNECT_PLATFORM "linux-arm64")+ endif()+endif()++if (LLAMA_CONNECT_PLATFORM STREQUAL "")+ message(WARNING "llama-connect: no prebuilt binary for ${CMAKE_SYSTEM_NAME}/${CMAKE_SYSTEM_PROCESSOR}, skipping")+ return()+endif()++if (LLAMA_CONNECT_PLATFORM MATCHES "^win-")+ set(LLAMA_CONNECT_BINARY "llama-connect.exe")+else()+ set(LLAMA_CONNECT_BINARY "llama-connect")+endif()++# multi-config generators put the binaries in a per-config subdir+get_property(LLAMA_CONNECT_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)+if (LLAMA_CONNECT_MULTI_CONFIG)+ set(LLAMA_CONNECT_OUT_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")+else()+ set(LLAMA_CONNECT_OUT_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")+endif()++# the script is a no-op if the binary is already downloaded+add_custom_target(${TARGET} ALL+ BYPRODUCTS "${LLAMA_CONNECT_OUT_DIR}/${LLAMA_CONNECT_BINARY}"+ COMMAND ${CMAKE_COMMAND}+ "-DREPO=${LLAMA_CONNECT_REPO}"+ "-DVERSION=${LLAMA_CONNECT_VERSION}"+ "-DPLATFORM=${LLAMA_CONNECT_PLATFORM}"+ "-DOUT_DIR=${LLAMA_CONNECT_OUT_DIR}"+ "-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}"+ -P "${PROJECT_SOURCE_DIR}/scripts/connect-download.cmake"+ COMMENT "Downloading llama-connect (${LLAMA_CONNECT_PLATFORM}, ${LLAMA_CONNECT_VERSION})"+ VERBATIM+)++# building llama-server should also pull the binary (for convenience)+if (TARGET llama-server-impl)+ add_dependencies(llama-server-impl ${TARGET})+endif()++if (LLAMA_TOOLS_INSTALL)+ install(PROGRAMS "${LLAMA_CONNECT_OUT_DIR}/${LLAMA_CONNECT_BINARY}" TYPE BIN)+endif()diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txtindex 70b8d7da8e..e0b9d0ec19 100644--- a/tools/server/CMakeLists.txt+++ b/tools/server/CMakeLists.txt@@ -41,6 +41,8 @@ set(TARGET llama-server-impl)add_library(${TARGET}server.cpp+ server-connect.cpp+ server-connect.hserver-http.cppserver-http.hserver-models.cppdiff --git a/tools/server/README.md b/tools/server/README.mdindex ec82db0076..e1c5804da5 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -208,6 +208,8 @@ For the full list of features, please refer to [server's changelog](https://gith| `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) || `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) || `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) |+| `--connect` | (experimental) open a peer-to-peer tunnel for server using llama-connect (default: disabled)<br/>(env: LLAMA_ARG_CONNECT) |+| `--connect-code CODE` | manually specify a 40-character code for --connect (default: generate a new one for each run)<br/>(env: LLAMA_ARG_CONNECT_CODE) || `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) || `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)<br/>(env: LLAMA_ARG_RERANKING) || `--api-key KEY` | API key to use for authentication, multiple keys can be provided as a comma-separated list (default: none)<br/>(env: LLAMA_API_KEY) |@@ -235,7 +237,7 @@ For the full list of features, please refer to [server's changelog](https://gith| `--reasoning-effort LEVEL` | reasoning effort level given to the chat template: 'default' to keep the template default,<br/>or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)<br/>(env: LLAMA_ARG_REASONING_EFFORT) || `--reasoning-budget N` | token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) || `--reasoning-budget-message MESSAGE` | message injected before the end-of-thinking tag when reasoning budget is exhausted (default: none)<br/>(env: LLAMA_ARG_THINK_BUDGET_MESSAGE) |-| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: enabled)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) |+| `--reasoning-preserve, --no-reasoning-preserve` | preserve reasoning trace in the full history, not just the last assistant message (default: template default)<br/>compatible with certain templates having 'supports_preserve_reasoning' capability<br/>example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking<br/>(env: LLAMA_ARG_REASONING_PRESERVE) || `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) || `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek-ocr, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, granite-4.0, granite-4.1, grok-2, hunyuan-dense, hunyuan-moe, hunyuan-vl, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) || `--skip-chat-parsing, --no-skip-chat-parsing` | force a pure content parser, even if a Jinja template is specified; model will output everything in the content section, including any reasoning and/or tool calls (default: disabled)<br/>(env: LLAMA_ARG_SKIP_CHAT_PARSING) |diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex 5577c339db..3ea20c3d2f 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -9,6 +9,7 @@#include "server-common.h"+#include <filesystem>#include <random>#include <sstream>#include <fstream>@@ -33,6 +34,60 @@#include <unistd.h>#endif+#if defined(_WIN32)+#define WIN32_LEAN_AND_MEAN+#ifndef NOMINMAX+# define NOMINMAX+#endif+#include <windows.h>+#elif defined(__APPLE__) && defined(__MACH__)+#include <mach-o/dyld.h>+#include <limits.h>+#else+#include <unistd.h>+#endif++std::filesystem::path get_server_exec_path() {+#if defined(_WIN32)+ wchar_t buf[32768] = { 0 }; // Large buffer to handle long paths+ DWORD len = GetModuleFileNameW(nullptr, buf, _countof(buf));+ if (len == 0 || len >= _countof(buf)) {+ throw std::runtime_error("GetModuleFileNameW failed or path too long");+ }+ return std::filesystem::path(buf);+#elif defined(__APPLE__) && defined(__MACH__)+ char small_path[PATH_MAX];+ uint32_t size = sizeof(small_path);++ if (_NSGetExecutablePath(small_path, &size) == 0) {+ // resolve any symlinks to get absolute path+ try {+ return std::filesystem::canonical(std::filesystem::path(small_path));+ } catch (...) {+ return std::filesystem::path(small_path);+ }+ } else {+ // buffer was too small, allocate required size and call again+ std::vector<char> buf(size);+ if (_NSGetExecutablePath(buf.data(), &size) == 0) {+ try {+ return std::filesystem::canonical(std::filesystem::path(buf.data()));+ } catch (...) {+ return std::filesystem::path(buf.data());+ }+ }+ throw std::runtime_error("_NSGetExecutablePath failed after buffer resize");+ }+#else+ char path[FILENAME_MAX];+ ssize_t count = readlink("/proc/self/exe", path, FILENAME_MAX);+ if (count <= 0) {+ throw std::runtime_error("failed to resolve /proc/self/exe");+ }+ return std::filesystem::path(std::string(path, count));+#endif+}+json format_error_response(const std::string & message, const enum error_type type) {std::string type_str;int code = 500;diff --git a/tools/server/server-common.h b/tools/server/server-common.hindex 5bd2ed57ab..2777521221 100644--- a/tools/server/server-common.h+++ b/tools/server/server-common.h@@ -15,6 +15,7 @@#include <condition_variable>#include <cinttypes>#include <cstdio>+#include <filesystem>#include <functional>#include <mutex>#include <queue>@@ -94,6 +95,9 @@ struct server_grammar_trigger {json format_error_response(const std::string & message, const enum error_type type);+// path of the running llama-server binary, used to spawn siblings. throws on failure+std::filesystem::path get_server_exec_path();+//// random string / id//diff --git a/tools/server/server-connect.cpp b/tools/server/server-connect.cppnew file mode 100644index 0000000000..2313e39dae--- /dev/null+++ b/tools/server/server-connect.cpp@@ -0,0 +1,297 @@+#include "server-connect.h"++#include "server-common.h"+#include "subproc.h"++#include <cctype>+#include <chrono>+#include <exception>+#include <filesystem>+#include <random>+#include <string_view>+#include <system_error>+#include <thread>+#include <vector>++// share code = room code + pass code, must match llama-connect and the Web UI+// ref: https://github.com/ggml-org/llama-connect/blob/master/src/protocol.rs+static constexpr size_t CONNECT_ROOM_CODE_LEN = 8;+static constexpr size_t CONNECT_PASS_CODE_LEN = 32;++static const std::string CONNECT_CODE_CHARS = "ABCDEFGHJKMNPQRSTUVWXYZabcdefghjkmnpqrstuvwxyz23456789";++#if defined(_WIN32)+static const std::string CONNECT_EXE_NAME = "llama-connect.exe";+static constexpr char PATH_SEPARATOR = ';';+#else+static const std::string CONNECT_EXE_NAME = "llama-connect";+static constexpr char PATH_SEPARATOR = ':';+#endif++// how long to wait for the child to notice the closed stdin before killing it+static constexpr int CONNECT_STOP_TIMEOUT_MS = 3000;++// the pass code guards the tunnel, so do not use random_string(): its mt19937 is predictable+static std::string gen_share_code() {+ std::random_device rd;+ std::uniform_int_distribution<size_t> dist(0, CONNECT_CODE_CHARS.size() - 1);++ std::string code(CONNECT_ROOM_CODE_LEN + CONNECT_PASS_CODE_LEN, ' ');+ for (char & c : code) {+ c = CONNECT_CODE_CHARS[dist(rd)];+ }++ return code;+}++static std::string format_share_code(const std::string & code) {+ std::string out;+ for (size_t i = 0; i < code.size(); i += 8) {+ if (i > 0) {+ out += ' ';+ }+ out += code.substr(i, 8);+ }+ return out;+}++// whitespace is tolerated, the Web UI shows the code in blocks and users paste it back+// returns an empty string if it is not a valid share code+static std::string normalize_share_code(const std::string & key) {+ std::string code;+ for (char c : key) {+ if (!std::isspace((unsigned char) c)) {+ code += c;+ }+ }++ if (code.size() != CONNECT_ROOM_CODE_LEN + CONNECT_PASS_CODE_LEN) {+ return "";+ }++ if (code.find_first_not_of(CONNECT_CODE_CHARS) != std::string::npos) {+ return "";+ }++ return code;+}++// llama-connect logs as "[LEVEL] text", forward at the same level so our verbosity filter applies+static void forward_child_log(const std::string & line) {+ static const std::pair<std::string_view, ggml_log_level> tags[] = {+ { "[ERROR] ", GGML_LOG_LEVEL_ERROR },+ { "[WARN] ", GGML_LOG_LEVEL_WARN },+ { "[INFO] ", GGML_LOG_LEVEL_INFO },+ { "[DEBUG] ", GGML_LOG_LEVEL_DEBUG },+ { "[TRACE] ", GGML_LOG_LEVEL_DEBUG },+ };++ // untagged lines are the startup banner, show them as info+ ggml_log_level level = GGML_LOG_LEVEL_INFO;+ const char * text = line.c_str();++ for (const auto & [tag, tag_level] : tags) {+ if (string_starts_with(line, tag)) {+ level = tag_level;+ text += tag.size();+ break;+ }+ }++ switch (level) {+ case GGML_LOG_LEVEL_ERROR: LOG_ERR("connect | %s", text); break;+ case GGML_LOG_LEVEL_WARN: LOG_WRN("connect | %s", text); break;+ case GGML_LOG_LEVEL_DEBUG: LOG_DBG("connect | %s", text); break;+ default: LOG_INF("connect | %s", text); break;+ }+}++// the child dials over TCP: a wildcard bind is reached on loopback, a specific address on itself+// empty when every listener is a unix socket+static std::string upstream_host(const std::vector<std::string> & hosts) {+ for (const auto & host : hosts) {+ if (string_ends_with(host, ".sock")) {+ continue;+ }+ if (host == "0.0.0.0") {+ return "127.0.0.1";+ }+ if (host == "::") {+ return "[::1]";+ }+ // llama-connect builds http://host:port, so an IPv6 literal needs brackets+ if (host.find(':') != std::string::npos && host[0] != '[') {+ return "[" + host + "]";+ }+ return host;+ }+ return "";+}++static bool path_is_file(const std::filesystem::path & p) {+ std::error_code ec;+ return std::filesystem::is_regular_file(p, ec);+}++std::string server_connect::find_binary() {+ // prefer the copy shipped next to llama-server over an unrelated one in PATH+ try {+ auto sibling = get_server_exec_path().parent_path() / CONNECT_EXE_NAME;+ if (path_is_file(sibling)) {+ return sibling.string();+ }+ } catch (const std::exception & e) {+ SRV_WRN("could not resolve the llama-server path (%s), looking for llama-connect in PATH only\n", e.what());+ }++ const std::string path_env = common_get_env("PATH");+ size_t start = 0;+ while (start <= path_env.size()) {+ size_t end = path_env.find(PATH_SEPARATOR, start);+ if (end == std::string::npos) {+ end = path_env.size();+ }+ const std::string dir = path_env.substr(start, end - start);+ if (!dir.empty()) {+ auto candidate = std::filesystem::path(dir) / CONNECT_EXE_NAME;+ if (path_is_file(candidate)) {+ return candidate.string();+ }+ }+ start = end + 1;+ }++ return "";+}++std::string server_connect::unavailable_reason(const common_params & params) {+ if (!common_subproc::is_supported()) {+ return "this build has subprocess support disabled, rebuild with -DLLAMA_SUBPROCESS=ON";+ }++ if (!params.server_connect_code.empty() && normalize_share_code(params.server_connect_code).empty()) {+ return "--connect-code must be " + std::to_string(CONNECT_ROOM_CODE_LEN + CONNECT_PASS_CODE_LEN)+ + " characters from '" + CONNECT_CODE_CHARS + "'";+ }++ if (upstream_host(params.hostnames).empty()) {+ return "llama-connect reaches the server over TCP, add a TCP address to --host";+ }++ if (find_binary().empty()) {+ return "could not find '" + CONNECT_EXE_NAME + "' next to llama-server or in PATH.\n"+ " it is a separate binary: download it from https://github.com/ggml-org/llama-connect/releases\n"+ " or build llama.cpp with -DLLAMA_CONNECT=ON to have it fetched automatically";+ }++ return "";+}++std::string server_connect::resolve_code(const std::string & code) {+ return code.empty() ? gen_share_code() : normalize_share_code(code);+}++bool server_connect::start(const common_params & params, int port) {+ const std::string bin = find_binary();+ if (bin.empty()) {+ SRV_ERR("%s", "llama-connect binary not found\n");+ return false;+ }++ // main() resolves this before anything can read it from /props+ const std::string & code = params.server_connect_code;+ GGML_ASSERT(!code.empty());++ const std::vector<std::string> args = {+ bin,+ "--host", upstream_host(params.hostnames),+ "--port", std::to_string(port),+ "--code", code,+ // the kernel closes our end of this pipe even if we are killed without cleanup+ // this is what stops the child from outliving us+ "--exit-on-stdin-eof",+ };++ proc = std::make_unique<common_subproc>();++ const int options = subprocess_option_no_window+ | subprocess_option_combined_stdout_stderr+ | subprocess_option_inherit_environment;++ if (!proc->create(args, options)) {+ SRV_ERR("failed to spawn '%s'\n", bin.c_str());+ proc.reset();+ return false;+ }++ log_thread = std::thread([this]() {+ FILE * out = proc->stdout_file();+ if (out == nullptr) {+ SRV_ERR("%s", "failed to get stdout of the llama-connect process\n");+ return;+ }+ std::vector<char> buf(4096);+ while (fgets(buf.data(), (int) buf.size(), out) != nullptr) {+ forward_child_log(buf.data());+ }+ // EOF means the child is gone+ if (!stopping.load(std::memory_order_acquire)) {+ SRV_ERR("%s", "llama-connect exited on its own, remote access is no longer available\n");+ }+ });++ SRV_INF("%s", "-----------------\n");+ SRV_INF("%s", "remote access is enabled via llama-connect\n");+ SRV_INF("share code (enter it in the Web UI under Settings -> Remote Access): %s\n",+ format_share_code(code).c_str());+ SRV_WRN("%s", "anyone with this code can use this server, do not share it publicly\n");+ SRV_INF("%s", "-----------------\n");++ return true;+}++void server_connect::stop() {+ if (!proc) {+ return;+ }++ SRV_INF("%s", "stopping llama-connect...\n");++ stopping.store(true, std::memory_order_release);++ proc->close_stdin();++ for (int elapsed = 0; elapsed < CONNECT_STOP_TIMEOUT_MS && proc->alive(); elapsed += 100) {+ std::this_thread::sleep_for(std::chrono::milliseconds(100));+ }++ // no-op if the child already exited. the log thread leaves fgets() on EOF, which the kernel+ // raises once every writer of the stdout pipe is gone, so llama-connect must not spawn+ // children of its own: one of them holding that pipe blocks the join below forever+ proc->terminate();++ if (log_thread.joinable()) {+ try {+ log_thread.join();+ } catch (const std::system_error & e) {+ // ~thread() on a still-joinable thread calls std::terminate, detach instead+ SRV_ERR("failed to join the llama-connect log thread: %s\n", e.what());+ log_thread.detach();+ }+ }++ proc->join(); // reap the zombie+ proc.reset();+}++server_connect::server_connect() = default;++server_connect::~server_connect() {+ try {+ stop();+ } catch (const std::exception & e) {+ SRV_ERR("failed to stop llama-connect: %s\n", e.what());+ } catch (...) {+ SRV_ERR("%s", "failed to stop llama-connect\n");+ }+}diff --git a/tools/server/server-connect.h b/tools/server/server-connect.hnew file mode 100644index 0000000000..a71f557b3c--- /dev/null+++ b/tools/server/server-connect.h@@ -0,0 +1,39 @@+#pragma once++#include "common.h"++#include <atomic>+#include <memory>+#include <string>+#include <thread>++struct common_subproc;++// spawns llama-connect, which exposes this server to a remote browser over WebRTC+// core binary is in a separate project: https://github.com/ggml-org/llama-connect+struct server_connect {+ server_connect();+ ~server_connect();++ server_connect(const server_connect &) = delete;+ server_connect & operator=(const server_connect &) = delete;++ // path of the llama-connect binary, empty if not found+ static std::string find_binary();++ // why --connect cannot work here, empty if it can+ static std::string unavailable_reason(const common_params & params);++ static std::string resolve_code(const std::string & code);++ // port is the bound one, it differs from params.port with --port 0 or the GCP override+ bool start(const common_params & params, int port);++ // idempotent, also called by the destructor+ void stop();++private:+ std::unique_ptr<common_subproc> proc;+ std::thread log_thread;+ std::atomic<bool> stopping{false}; // tells the log thread the exit is expected+};diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 9c3c54c3a6..b2116b1740 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -5264,6 +5264,10 @@ static json get_res_props(const server_context_meta & meta, const common_params{ "is_sleeping", is_sleeping },{ "cors_proxy_enabled", params.ui_mcp_proxy },};++ if (!params.server_connect_code.empty()) {+ props["connect_code"] = params.server_connect_code;+ }if (params.use_jinja) {if (!tmpl_tools.empty()) {props["chat_template_tool_use"] = tmpl_tools;diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex d77837c866..606eab4f4a 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -35,12 +35,6 @@extern char **environ;#endif-#if defined(__APPLE__) && defined(__MACH__)-// macOS: use _NSGetExecutablePath to get the executable path-#include <mach-o/dyld.h>-#include <limits.h>-#endif-#define DEFAULT_STOP_TIMEOUT 10 // seconds#define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit"@@ -436,47 +430,6 @@ struct server_lru_sched {// delete). distinct from params.timeout_read/write which only applies to the generation proxystatic constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250;-static std::filesystem::path get_server_exec_path() {-#if defined(_WIN32)- wchar_t buf[32768] = { 0 }; // Large buffer to handle long paths- DWORD len = GetModuleFileNameW(nullptr, buf, _countof(buf));- if (len == 0 || len >= _countof(buf)) {- throw std::runtime_error("GetModuleFileNameW failed or path too long");- }- return std::filesystem::path(buf);-#elif defined(__APPLE__) && defined(__MACH__)- char small_path[PATH_MAX];- uint32_t size = sizeof(small_path);-- if (_NSGetExecutablePath(small_path, &size) == 0) {- // resolve any symlinks to get absolute path- try {- return std::filesystem::canonical(std::filesystem::path(small_path));- } catch (...) {- return std::filesystem::path(small_path);- }- } else {- // buffer was too small, allocate required size and call again- std::vector<char> buf(size);- if (_NSGetExecutablePath(buf.data(), &size) == 0) {- try {- return std::filesystem::canonical(std::filesystem::path(buf.data()));- } catch (...) {- return std::filesystem::path(buf.data());- }- }- throw std::runtime_error("_NSGetExecutablePath failed after buffer resize");- }-#else- char path[FILENAME_MAX];- ssize_t count = readlink("/proc/self/exe", path, FILENAME_MAX);- if (count <= 0) {- throw std::runtime_error("failed to resolve /proc/self/exe");- }- return std::filesystem::path(std::string(path, count));-#endif-}-static void unset_reserved_args(common_preset & preset, bool unset_model_args) {preset.unset_option("LLAMA_ARG_SSL_KEY_FILE");preset.unset_option("LLAMA_ARG_SSL_CERT_FILE");@@ -486,6 +439,8 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) {preset.unset_option("LLAMA_ARG_MODELS_MAX");preset.unset_option("LLAMA_ARG_MODELS_PRESET");preset.unset_option("LLAMA_ARG_MODELS_AUTOLOAD");+ preset.unset_option("LLAMA_ARG_CONNECT");+ preset.unset_option("LLAMA_ARG_CONNECT_CODE");if (unset_model_args) {preset.unset_option("LLAMA_ARG_MODEL");preset.unset_option("LLAMA_ARG_MMPROJ");@@ -2282,7 +2237,7 @@ void server_models_routes::init_routes() {if (name.empty()) {// main instanceauto res = std::make_unique<server_http_res>();- res_ok(res, {+ json props = {// TODO: add support for this on web UI{"role", "router"},{"max_instances", params.models_max},@@ -2298,7 +2253,11 @@ void server_models_routes::init_routes() {{"ui_settings", ui_settings},{"build_info", std::string(llama_build_info())},{"cors_proxy_enabled", params.ui_mcp_proxy},- });+ };+ if (!params.server_connect_code.empty()) {+ props["connect_code"] = params.server_connect_code;+ }+ res_ok(res, props);return res;}return proxy_get(req);diff --git a/tools/server/server.cpp b/tools/server/server.cppindex 871e827bef..b168241076 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -1,3 +1,4 @@+#include "server-connect.h"#include "server-context.h"#include "server-http.h"#include "server-models.h"@@ -191,6 +192,20 @@ int llama_server(common_params & params, int argc, char ** argv, server_child ¶ms.model_alias.insert(model_name);}+ // check early, so a missing llama-connect fails-fast+ if (params.server_connect) {+ const std::string reason = server_connect::unavailable_reason(params);+ if (!reason.empty()) {+ SRV_ERR("--connect is not available: %s\n", reason.c_str());+ return 1;+ }+ params.server_connect_code = server_connect::resolve_code(params.server_connect_code);+ } else if (!params.server_connect_code.empty()) {+ // the code is only meaningful while the tunnel runs, and /props keys off it being set+ SRV_WRN("%s", "--connect-code is ignored without --connect\n");+ params.server_connect_code.clear();+ }+// note: this is guaranteed to out-live ctx_http and toolsserver_mcp mcp_mgr;@@ -347,6 +362,10 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &warn_names.push_back("router mode");}+ if (params.server_connect) {+ warn_names.push_back("peer-to-peer tunnel (llama-connect, experimental)");+ }+if (params.ui_mcp_proxy) {ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));@@ -525,6 +544,16 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &SRV_INF("listening on %s\n", address.c_str());}+ // spawn only once listening, so the port is bound and the child startup health check reaches us. the destructor stops it+ server_connect connect_proc;+ if (params.server_connect && !connect_proc.start(params, ctx_http.port)) {+ SRV_ERR("%s", "exiting due to llama-connect error\n");+ ctx_http.stop();+ ctx_http.join();+ clean_up();+ return 1;+ }+if (is_router_server) {if (!params.models_preset_hf.empty()) {SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());