Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlx: deduplicate CUDA runtime payloads (#17956)
Daniel Hiltgen2cd63f7a1503
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yamlindex 8517dcd1e..58acb9443 100644--- a/.github/workflows/release.yaml+++ b/.github/workflows/release.yaml@@ -659,9 +659,6 @@ jobs:build-args: ${{ matrix.build-args }}outputs: type=local,dest=dist/${{ matrix.os }}-${{ matrix.arch }}cache-from: ${{ matrix.cache-from }}- - name: Deduplicate CUDA libraries- run: |- ./scripts/deduplicate_cuda_libs.sh dist/${{ matrix.os }}-${{ matrix.arch }}- name: Verify Linux build payloadsshell: bashrun: |@@ -673,6 +670,24 @@ jobs:do[ -f "$payload" ] || { echo "missing $payload"; exit 1; }done+ if [ "${{ matrix.arch }}" = "amd64" ] && [ "${{ matrix.suffix }}" != "-rocm" ]; then+ for payload in \+ "$base/lib/ollama/mlx_cuda_v13/libopenblas.so.0" \+ "$base/lib/ollama/mlx_cuda_v13/libgfortran.so.5" \+ "$base/lib/ollama/mlx_cuda_v13/libquadmath.so.0"+ do+ [ -f "$payload" ] || { echo "missing $payload"; exit 1; }+ done+ mlx_deps="$(ldd "$base/lib/ollama/mlx_cuda_v13/libmlxc.so")"+ echo "$mlx_deps"+ for dependency in libgfortran.so.5 libquadmath.so.0+ do+ echo "$mlx_deps" | grep -E "^[[:space:]]*$dependency => .*mlx_cuda_v13/" >/dev/null || {+ echo "$dependency does not resolve from the MLX payload"+ exit 1+ }+ done+ fi- run: |for COMPONENT in bin/* lib/ollama/*; docase "$COMPONENT" indiff --git a/.github/workflows/test-llamacpp-update.yaml b/.github/workflows/test-llamacpp-update.yamlindex 542c766e2..81ba9db7b 100644--- a/.github/workflows/test-llamacpp-update.yaml+++ b/.github/workflows/test-llamacpp-update.yaml@@ -218,7 +218,6 @@ jobs:copy_payload cuda_jetpack6fi- ./scripts/deduplicate_cuda_libs.sh "${dest}"- name: Verify Linux build payloadsshell: bashrun: |diff --git a/cmake/local.cmake b/cmake/local.cmakeindex b2dad3adb..3c7d18f2b 100644--- a/cmake/local.cmake+++ b/cmake/local.cmake@@ -12,6 +12,20 @@ set(_ollama_mlx_backends_doc "Semicolon-separated MLX backends to build: cuda_v1set(OLLAMA_VERSION "0.0.0" CACHE STRING "Ollama version embedded in the local Go binary")set(OLLAMA_PAYLOAD_INSTALL_PREFIX "${CMAKE_BINARY_DIR}" CACHE PATH"Build-time staging prefix for nested Ollama native payloads")+set(OLLAMA_MLX_PAYLOAD_INSTALL_PREFIX "" CACHE PATH+ "Optional separate staging prefix for the MLX overlay payload")++get_filename_component(_ollama_mlx_base_install_prefix+ "${OLLAMA_PAYLOAD_INSTALL_PREFIX}" ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}")+set(_ollama_mlx_payload_install_prefix "${_ollama_mlx_base_install_prefix}")+set(_ollama_mlx_overlay_package OFF)+if(OLLAMA_MLX_PAYLOAD_INSTALL_PREFIX)+ get_filename_component(_ollama_mlx_payload_install_prefix+ "${OLLAMA_MLX_PAYLOAD_INSTALL_PREFIX}" ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}")+ if(NOT "${_ollama_mlx_payload_install_prefix}" STREQUAL "${_ollama_mlx_base_install_prefix}")+ set(_ollama_mlx_overlay_package ON)+ endif()+endif()string(REGEX REPLACE "^v" "" OLLAMA_VERSION "${OLLAMA_VERSION}")@@ -102,6 +116,12 @@ else()set(OLLAMA_MLX_BACKENDS "${OLLAMA_MLX_BACKENDS}" CACHE STRING "${_ollama_mlx_backends_doc}")endif()+if(OLLAMA_MLX_BACKENDS AND _ollama_mlx_overlay_package)+ add_custom_target(ollama-mlx-overlay-clean+ COMMAND ${CMAKE_COMMAND} -E rm -rf "${_ollama_mlx_payload_install_prefix}"+ COMMENT "Cleaning MLX overlay staging directory")+endif()+if(NOT OLLAMA_HAVE_LLAMA_SERVER)if(OLLAMA_LLAMA_BACKENDS)message(FATAL_ERROR "llama/server is required when OLLAMA_LLAMA_BACKENDS is set")@@ -528,13 +548,14 @@ function(ollama_add_mlx_build name)ollama_collect_cache_args_with_prefix("MLX_" _mlx_cache_args)set(_cmake_args-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}- -DCMAKE_INSTALL_PREFIX=${OLLAMA_PAYLOAD_INSTALL_PREFIX}+ -DCMAKE_INSTALL_PREFIX=${_ollama_mlx_base_install_prefix}-DOLLAMA_LIB_DIR:STRING=${OLLAMA_LIB_DIR}-DOLLAMA_RUNNER_DIR=${ARG_RUNNER_DIR}-DOLLAMA_SOURCE_DIR=${CMAKE_SOURCE_DIR}-DFETCHCONTENT_SOURCE_DIR_MLX=${OLLAMA_MLX_SOURCE_DIR}-DFETCHCONTENT_SOURCE_DIR_MLX-C=${OLLAMA_MLX_C_SOURCE_DIR}-DOLLAMA_MLX_GENERATE_WRAPPERS=OFF+ -DOLLAMA_MLX_OVERLAY_PACKAGE=${_ollama_mlx_overlay_package}${ARG_CMAKE_ARGS}${_mlx_cache_args})@@ -575,6 +596,26 @@ function(ollama_add_mlx_build name)${_cmake_args})endif()+ set(_install_command)+ list(APPEND _install_command+ ${CMAKE_COMMAND} --install <BINARY_DIR>+ ${OLLAMA_NATIVE_CONFIG_ARG}+ --prefix "${_ollama_mlx_payload_install_prefix}"+ --component MLX+ COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>+ ${OLLAMA_NATIVE_CONFIG_ARG}+ --prefix "${_ollama_mlx_payload_install_prefix}"+ --component MLX_VENDOR+ COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>+ ${OLLAMA_NATIVE_CONFIG_ARG}+ --prefix "${_ollama_mlx_base_install_prefix}"+ --component MLX_SHARED)++ set(_mlx_dependencies ollama-mlx-sources)+ if(_ollama_mlx_overlay_package)+ list(APPEND _mlx_dependencies ollama-mlx-overlay-clean)+ endif()+ExternalProject_Add(ollama-mlx-${name}SOURCE_DIR ${CMAKE_SOURCE_DIR}/cmake/mlxBINARY_DIR ${_build_dir}@@ -584,13 +625,8 @@ function(ollama_add_mlx_build name)${OLLAMA_NATIVE_BUILD_TARGET_ARG} mlx${OLLAMA_NATIVE_BUILD_TARGET_ARG} mlxc${OLLAMA_NATIVE_BUILD_TARGET_ARG} ollama_xgrammar- INSTALL_COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>- ${OLLAMA_NATIVE_CONFIG_ARG}- --component MLX- COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>- ${OLLAMA_NATIVE_CONFIG_ARG}- --component MLX_VENDOR- DEPENDS ollama-mlx-sources+ INSTALL_COMMAND ${_install_command}+ DEPENDS ${_mlx_dependencies}LIST_SEPARATOR |BUILD_ALWAYS TRUE${OLLAMA_NATIVE_EXTERNAL_OPTIONS}diff --git a/cmake/mlx/CMakeLists.txt b/cmake/mlx/CMakeLists.txtindex d1f880158..ade40df92 100644--- a/cmake/mlx/CMakeLists.txt+++ b/cmake/mlx/CMakeLists.txt@@ -19,16 +19,6 @@ set(CMAKE_CXX_STANDARD 17)set(CMAKE_CXX_STANDARD_REQUIRED ON)set(CMAKE_CXX_EXTENSIONS ON)-if(APPLE)- set(CMAKE_BUILD_RPATH "@loader_path")- set(CMAKE_INSTALL_RPATH "@loader_path")- set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)-elseif(UNIX)- set(CMAKE_BUILD_RPATH "$ORIGIN")- set(CMAKE_INSTALL_RPATH "$ORIGIN")- set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)-endif()-if(NOT DEFINED OLLAMA_SOURCE_DIR OR "${OLLAMA_SOURCE_DIR}" STREQUAL "")get_filename_component(OLLAMA_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE)endif()@@ -39,6 +29,42 @@ set(OLLAMA_LIB_DIR "lib/ollama" CACHE STRING "Install destination for Ollama runset(OLLAMA_RUNNER_DIR "" CACHE STRING "Ollama runtime payload subdirectory")set(OLLAMA_BUILD_DIR ${CMAKE_BINARY_DIR}/lib/ollama)set(OLLAMA_INSTALL_DIR ${OLLAMA_LIB_DIR}/${OLLAMA_RUNNER_DIR})+set(OLLAMA_DEPENDENCY_INSTALL_DIR "")+set(OLLAMA_DEPENDENCY_RUNNER_DIR "")+if(OLLAMA_RUNNER_DIR MATCHES "^mlx_(cuda.*)$")+ set(OLLAMA_DEPENDENCY_RUNNER_DIR "${CMAKE_MATCH_1}")+ set(OLLAMA_DEPENDENCY_INSTALL_DIR "${OLLAMA_LIB_DIR}/${OLLAMA_DEPENDENCY_RUNNER_DIR}")+endif()++set(OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS "${OLLAMA_INSTALL_DIR}/include")+set(OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR "${OLLAMA_INSTALL_DIR}")+set(OLLAMA_MLX_PRIVATE_CUDA_INSTALL_DIR "${OLLAMA_INSTALL_DIR}")+set(OLLAMA_MLX_CUDNN_INSTALL_DIR "${OLLAMA_INSTALL_DIR}")+set(OLLAMA_MLX_RPATH)++if(APPLE)+ set(OLLAMA_MLX_RPATH "@loader_path")+elseif(WIN32)+ if(OLLAMA_DEPENDENCY_INSTALL_DIR)+ set(OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR "${OLLAMA_DEPENDENCY_INSTALL_DIR}")+ set(OLLAMA_MLX_PRIVATE_CUDA_INSTALL_DIR "${OLLAMA_DEPENDENCY_INSTALL_DIR}")+ set(MLX_CUDA_BIN_DIR "../${OLLAMA_DEPENDENCY_RUNNER_DIR}" CACHE STRING "" FORCE)+ set(MLX_CUDNN_BIN_DIR "." CACHE STRING "" FORCE)+ endif()+elseif(UNIX)+ set(OLLAMA_MLX_RPATH "$ORIGIN")+ list(APPEND OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS "${OLLAMA_LIB_DIR}/include")+ if(OLLAMA_DEPENDENCY_INSTALL_DIR)+ set(OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR "${OLLAMA_DEPENDENCY_INSTALL_DIR}")+ list(APPEND OLLAMA_MLX_RPATH "$ORIGIN/../${OLLAMA_DEPENDENCY_RUNNER_DIR}")+ endif()+endif()++if(OLLAMA_MLX_RPATH)+ set(CMAKE_BUILD_RPATH ${OLLAMA_MLX_RPATH})+ set(CMAKE_INSTALL_RPATH ${OLLAMA_MLX_RPATH})+ set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)+endif()set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OLLAMA_BUILD_DIR})set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${OLLAMA_BUILD_DIR})@@ -52,6 +78,7 @@ if(MLX_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES)endif()option(OLLAMA_MLX_GENERATE_WRAPPERS "Regenerate MLX Go wrappers" OFF)+option(OLLAMA_MLX_OVERLAY_PACKAGE "Install the MLX payload as an overlay package" OFF)message(STATUS "Setting up MLX (this takes a while...)")foreach(_cudnn_var CUDNN_INCLUDE_PATH CUDNN_LIBRARY_PATH)@@ -61,6 +88,20 @@ foreach(_cudnn_var CUDNN_INCLUDE_PATH CUDNN_LIBRARY_PATH)endforeach()add_subdirectory(${OLLAMA_SOURCE_DIR}/mlx ${CMAKE_BINARY_DIR}/mlx)+if(UNIX AND NOT APPLE)+ foreach(_mlx_target mlx mlxc)+ if(TARGET ${_mlx_target})+ set_target_properties(${_mlx_target} PROPERTIES+ BUILD_RPATH "${OLLAMA_MLX_RPATH}"+ INSTALL_RPATH "${OLLAMA_MLX_RPATH}"+ BUILD_WITH_INSTALL_RPATH ON)+ # DT_RPATH is inherited by transitive dependencies such as+ # OpenBLAS -> libgfortran -> libquadmath; DT_RUNPATH is not.+ target_link_options(${_mlx_target} PRIVATE LINKER:--disable-new-dtags)+ endif()+ endforeach()+endif()+include(FetchContent)set(XGRAMMAR_VERSION v0.2.7)FetchContent_Declare(xgrammar@@ -93,7 +134,7 @@ add_subdirectory(${OLLAMA_SOURCE_DIR}/mlxrunner/xgrammar/native ${CMAKE_BINARY_Dfind_package(CUDAToolkit)# Build list of directories for runtime dependency resolution.-set(MLX_RUNTIME_DIRS ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_BIN_DIR}/x64 ${CUDAToolkit_LIBRARY_DIR})+set(MLX_RUNTIME_SEARCH_DIRS ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_BIN_DIR}/x64 ${CUDAToolkit_LIBRARY_DIR})# Add cuDNN bin paths for DLLs (Windows MLX CUDA builds).# CUDNN_ROOT_DIR is the standard CMake variable for cuDNN location.if(CUDNN_ROOT_DIR)@@ -105,24 +146,62 @@ if(_cudnn_root)file(TO_CMAKE_PATH "${_cudnn_root}" _cudnn_root)# cuDNN 9.x has versioned subdirectories under bin/ (e.g., bin/13.0/).file(GLOB CUDNN_BIN_SUBDIRS "${_cudnn_root}/bin/*")- list(APPEND MLX_RUNTIME_DIRS ${CUDNN_BIN_SUBDIRS})+ list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CUDNN_BIN_SUBDIRS})endif()# Add build output directory and MLX dependency build directories.-list(APPEND MLX_RUNTIME_DIRS ${OLLAMA_BUILD_DIR})+list(APPEND MLX_RUNTIME_SEARCH_DIRS ${OLLAMA_BUILD_DIR})+if(UNIX AND NOT APPLE)+ list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES})+endif()# OpenBLAS DLL location (pre-built zip extracts into openblas-src/bin/).-list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/openblas-src/bin)+list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CMAKE_BINARY_DIR}/_deps/openblas-src/bin)# NCCL: on Linux, if real NCCL is found, cmake bundles libnccl.so via the# regex below. If NCCL is not found, MLX links a static stub (OBJECT lib)# so there is no runtime dependency. This path covers the stub build dir# for windows so we include the DLL in our dependencies.-list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/distributed/nccl/nccl_stub-prefix/src/nccl_stub-build/Release)+list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/distributed/nccl/nccl_stub-prefix/src/nccl_stub-build/Release)# Non-link-time deps stay explicit; link-time deps derive from the mlx target.-set(MLX_INCLUDE_REGEXES cublas cublasLt cudart cufft nvrtc nvrtc-builtins cudnn nccl cusolver cusparse nv[Jj]it[Ll]ink openblas gfortran)+set(MLX_INCLUDE_REGEXES cublas cublasLt cudart cufft nvrtc nvrtc-builtins cudnn nccl cusolver cusparse nv[Jj]it[Ll]ink openblas gfortran quadmath)# On Windows, also include dl.dll (dlfcn-win32 POSIX emulation layer).if(WIN32)list(APPEND MLX_INCLUDE_REGEXES "^dl\\.dll$")endif()+set(MLX_RUNTIME_DEPENDENCY_FILTERS+ DIRECTORIES ${MLX_RUNTIME_SEARCH_DIRS}+ PRE_INCLUDE_REGEXES ${MLX_INCLUDE_REGEXES}+ PRE_EXCLUDE_REGEXES ".*")+set(MLX_EXPLICIT_CUDA_RUNTIME_REGEXES)+if(WIN32)+ list(APPEND MLX_EXPLICIT_CUDA_RUNTIME_REGEXES+ "[/\\\\]cublas[^/\\\\]*\\.dll$"+ "[/\\\\]cudart[^/\\\\]*\\.dll$"+ "[/\\\\]cudnn[^/\\\\]*\\.dll$"+ "[/\\\\]cufft[^/\\\\]*\\.dll$"+ "[/\\\\]cusolver[^/\\\\]*\\.dll$"+ "[/\\\\]cusparse[^/\\\\]*\\.dll$"+ "[/\\\\]nvfatbin[^/\\\\]*\\.dll$"+ "[/\\\\][nN][vV][jJ][iI][tT][lL][iI][nN][kK][^/\\\\]*\\.dll$"+ "[/\\\\]nvrtc[^/\\\\]*\\.dll$"+ "[/\\\\]nvrtc-builtins[^/\\\\]*\\.dll$")+elseif(UNIX AND NOT APPLE)+ list(APPEND MLX_EXPLICIT_CUDA_RUNTIME_REGEXES+ "[/\\\\]libcublas\\.so(\\..*)?$"+ "[/\\\\]libcublasLt\\.so(\\..*)?$"+ "[/\\\\]libcudart\\.so(\\..*)?$"+ "[/\\\\]libcudnn[^/\\\\]*\\.so(\\..*)?$"+ "[/\\\\]libcufft\\.so(\\..*)?$"+ "[/\\\\]libcusolver\\.so(\\..*)?$"+ "[/\\\\]libcusparse\\.so(\\..*)?$"+ "[/\\\\]libnvfatbin\\.so(\\..*)?$"+ "[/\\\\]libnvJitLink\\.so(\\..*)?$"+ "[/\\\\]libnvrtc\\.so(\\..*)?$"+ "[/\\\\]libnvrtc-builtins\\.so(\\..*)?$")+endif()+if(MLX_EXPLICIT_CUDA_RUNTIME_REGEXES)+ list(APPEND MLX_RUNTIME_DEPENDENCY_FILTERS+ POST_EXCLUDE_REGEXES ${MLX_EXPLICIT_CUDA_RUNTIME_REGEXES})+endif()# Keep mlx/mlxc targets separate from runtime dependencies so --strip only# applies to the binaries we build, not vendor DLLs/libs.@@ -198,9 +277,7 @@ install(CODE "" COMPONENT MLX)install(RUNTIME_DEPENDENCY_SET mlx_runtime_deps- DIRECTORIES ${MLX_RUNTIME_DIRS}- PRE_INCLUDE_REGEXES ${MLX_INCLUDE_REGEXES}- PRE_EXCLUDE_REGEXES ".*"+ ${MLX_RUNTIME_DEPENDENCY_FILTERS}RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDORLIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDOR)@@ -222,21 +299,6 @@ if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")COMPONENT MLX)endif()-# Install headers for NVRTC JIT compilation at runtime.-# MLX's own install rules use the default component so they get skipped by-# --component MLX. Headers are installed alongside libmlx in OLLAMA_INSTALL_DIR.-#-# Layout:-# ${OLLAMA_INSTALL_DIR}/include/cccl/ - CCCL headers-# ${OLLAMA_INSTALL_DIR}/include/{cute,cutlass}/ - CUTLASS/CUTE headers-# ${OLLAMA_INSTALL_DIR}/include/ - CUDA runtime/core headers-#-# MLX's jit_module.cpp resolves JIT support headers from the backend-local-# include directory. On Linux it also probes current_binary_dir().parent_path()-# / "include", so we create a symlink from lib/ollama/include to the backend-# include directory for archive packaging.-# This will need refinement if we add multiple CUDA versions for MLX in the future.-# CUDA runtime headers are found via CUDA_PATH env var (set by mlxrunner).set(_mlx_jit_cccl_include_dir "")if(CUDAToolkit_FOUND)foreach(_dir ${CUDAToolkit_INCLUDE_DIRS})@@ -252,28 +314,25 @@ endif()if(_mlx_jit_cccl_include_dir)foreach(_cccl_dir cuda nv cub thrust)if(EXISTS "${_mlx_jit_cccl_include_dir}/${_cccl_dir}")- install(DIRECTORY "${_mlx_jit_cccl_include_dir}/${_cccl_dir}"- DESTINATION ${OLLAMA_INSTALL_DIR}/include/cccl- COMPONENT MLX)+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)+ install(DIRECTORY "${_mlx_jit_cccl_include_dir}/${_cccl_dir}"+ DESTINATION ${_dst}/cccl+ COMPONENT MLX)+ endforeach()endif()endforeach()endif()if(EXISTS ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cute)- install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cute- DESTINATION ${OLLAMA_INSTALL_DIR}/include- COMPONENT MLX)- install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cutlass- DESTINATION ${OLLAMA_INSTALL_DIR}/include- COMPONENT MLX)+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)+ install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cute+ DESTINATION ${_dst}+ COMPONENT MLX)+ install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cutlass+ DESTINATION ${_dst}+ COMPONENT MLX)+ endforeach()endif()-# Install CUDA runtime/core headers needed by MLX JIT kernels.-# NVIDIA's NVRTC bundled-header model is CUDA Runtime + CCCL, not the entire-# toolkit include tree. Keep CCCL coherent above, include CUTLASS/CUTE above,-# and avoid shipping unrelated SDK headers such as NPP, CUPTI, cuRAND, NVML,-# cuBLAS, cuSPARSE, and cuSOLVER.-# The Go mlxrunner sets CUDA_PATH to OLLAMA_INSTALL_DIR so MLX finds them at-# $CUDA_PATH/include via NVRTC --include-path.if(CUDAToolkit_FOUND)# CUDAToolkit_INCLUDE_DIRS may be a semicolon-separated list# (e.g. ".../include;.../include/cccl"). Find the entry that@@ -288,8 +347,6 @@ if(CUDAToolkit_FOUND)if(NOT _cuda_inc)message(WARNING "Could not find cuda_runtime_api.h in CUDAToolkit_INCLUDE_DIRS: ${CUDAToolkit_INCLUDE_DIRS}")else()- set(_dst "${OLLAMA_INSTALL_DIR}/include")-set(_mlx_jit_cuda_headersbuiltin_types.hchannel_descriptor.h@@ -367,28 +424,23 @@ if(CUDAToolkit_FOUND)endif()endforeach()if(_mlx_jit_cuda_header_paths)- install(FILES ${_mlx_jit_cuda_header_paths}- DESTINATION ${_dst}- COMPONENT MLX)+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)+ install(FILES ${_mlx_jit_cuda_header_paths}+ DESTINATION ${_dst}+ COMPONENT MLX)+ endforeach()endif()foreach(_runtime_dir cooperative_groups crt)if(EXISTS "${_cuda_inc}/${_runtime_dir}")- install(DIRECTORY "${_cuda_inc}/${_runtime_dir}"- DESTINATION ${_dst}- COMPONENT MLX)+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)+ install(DIRECTORY "${_cuda_inc}/${_runtime_dir}"+ DESTINATION ${_dst}+ COMPONENT MLX)+ endforeach()endif()endforeach()- if(NOT WIN32 AND NOT APPLE)- install(CODE "- set(_link \"${CMAKE_INSTALL_PREFIX}/${OLLAMA_LIB_DIR}/include\")- set(_target \"${OLLAMA_RUNNER_DIR}/include\")- if(NOT EXISTS \${_link})- execute_process(COMMAND \${CMAKE_COMMAND} -E create_symlink \${_target} \${_link})- endif()- " COMPONENT MLX)- endif()endif()endif()@@ -403,46 +455,85 @@ if(WIN32 AND TARGET dl)COMPONENT MLX)endif()-# dlopen'd runtime libs, derived from MLX's dynamic.c/delayload.cppif(CUDAToolkit_FOUND)- file(GLOB MLX_CUDA_LIBS- "${CUDAToolkit_LIBRARY_DIR}/libcudart.so*"- "${CUDAToolkit_LIBRARY_DIR}/libcublas.so*"- "${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so*"- "${CUDAToolkit_LIBRARY_DIR}/libnvrtc.so*"- "${CUDAToolkit_LIBRARY_DIR}/libnvrtc-builtins.so*"- "${CUDAToolkit_LIBRARY_DIR}/libcufft.so*"- "${CUDAToolkit_LIBRARY_DIR}/libcusolver.so*"- "${CUDAToolkit_LIBRARY_DIR}/libcusparse.so*"- "${CUDAToolkit_LIBRARY_DIR}/libnvJitLink.so*"- "${CUDAToolkit_LIBRARY_DIR}/libcudnn*.so*")+ set(MLX_SHARED_CUDA_LIBS)+ set(MLX_PRIVATE_CUDA_LIBS)+ set(MLX_CUDNN_LIBS)if(WIN32)- file(GLOB MLX_CUDA_DLLS+ file(GLOB MLX_SHARED_CUDA_LIBS+ "${CUDAToolkit_BIN_DIR}/cublas64_*.dll"+ "${CUDAToolkit_BIN_DIR}/cublasLt64_*.dll"+ "${CUDAToolkit_BIN_DIR}/cudart64_*.dll"+ "${CUDAToolkit_BIN_DIR}/nvfatbin_*.dll"+ "${CUDAToolkit_BIN_DIR}/nvJitLink_*.dll"+ "${CUDAToolkit_BIN_DIR}/nvrtc64_*.dll""${CUDAToolkit_BIN_DIR}/nvrtc-builtins64_*.dll"- "${CUDAToolkit_BIN_DIR}/x64/nvrtc-builtins64_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/cublas64_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/cublasLt64_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/cudart64_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/nvfatbin_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/nvJitLink_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/nvrtc64_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/nvrtc-builtins64_*.dll")+ file(GLOB MLX_PRIVATE_CUDA_LIBS+ "${CUDAToolkit_BIN_DIR}/cufft64_*.dll""${CUDAToolkit_BIN_DIR}/cusolver64_*.dll"- "${CUDAToolkit_BIN_DIR}/x64/cusolver64_*.dll""${CUDAToolkit_BIN_DIR}/cusparse64_*.dll"- "${CUDAToolkit_BIN_DIR}/x64/cusparse64_*.dll"- "${CUDAToolkit_BIN_DIR}/nvJitLink_*.dll"- "${CUDAToolkit_BIN_DIR}/x64/nvJitLink_*.dll")- list(APPEND MLX_CUDA_LIBS ${MLX_CUDA_DLLS})+ "${CUDAToolkit_BIN_DIR}/x64/cufft64_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/cusolver64_*.dll"+ "${CUDAToolkit_BIN_DIR}/x64/cusparse64_*.dll")+ elseif(UNIX AND NOT APPLE)+ file(GLOB MLX_SHARED_CUDA_LIBS+ "${CUDAToolkit_LIBRARY_DIR}/libcudart.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libcublas.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libnvrtc.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libnvrtc-builtins.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libnvfatbin.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libnvJitLink.so*")+ file(GLOB MLX_PRIVATE_CUDA_LIBS+ "${CUDAToolkit_LIBRARY_DIR}/libcufft.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libcusolver.so*"+ "${CUDAToolkit_LIBRARY_DIR}/libcusparse.so*")+ endif()+ if(MLX_SHARED_CUDA_LIBS)+ list(REMOVE_DUPLICATES MLX_SHARED_CUDA_LIBS)+ install(FILES ${MLX_SHARED_CUDA_LIBS}+ DESTINATION ${OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR}+ COMPONENT MLX_SHARED)endif()- find_library(MLX_CUDNN_LIBRARY NAMES cudnn HINTS "$ENV{CUDNN_LIBRARY_PATH}")- if(MLX_CUDNN_LIBRARY)- get_filename_component(MLX_CUDNN_LIBRARY_DIR "${MLX_CUDNN_LIBRARY}" DIRECTORY)- file(GLOB MLX_CUDNN_LIBS "${MLX_CUDNN_LIBRARY_DIR}/libcudnn*.so*")- list(APPEND MLX_CUDA_LIBS ${MLX_CUDNN_LIBS})+ if(MLX_PRIVATE_CUDA_LIBS)+ list(REMOVE_DUPLICATES MLX_PRIVATE_CUDA_LIBS)+ install(FILES ${MLX_PRIVATE_CUDA_LIBS}+ DESTINATION ${OLLAMA_MLX_PRIVATE_CUDA_INSTALL_DIR}+ COMPONENT MLX_VENDOR)+ endif()+ if(UNIX AND NOT APPLE)+ find_library(MLX_CUDNN_LIBRARY NAMES cudnn+ HINTS "${CUDNN_LIBRARY_PATH}" "$ENV{CUDNN_LIBRARY_PATH}")+ if(MLX_CUDNN_LIBRARY)+ get_filename_component(MLX_CUDNN_LIBRARY_DIR "${MLX_CUDNN_LIBRARY}" DIRECTORY)+ file(GLOB MLX_CUDNN_LIBS "${MLX_CUDNN_LIBRARY_DIR}/libcudnn*.so*")+ endif()endif()if(WIN32 AND _cudnn_root)file(GLOB MLX_CUDNN_DLLS"${_cudnn_root}/bin/${CUDAToolkit_VERSION_MAJOR}.0/cudnn*.dll""${_cudnn_root}/bin/x64/cudnn*.dll")- list(APPEND MLX_CUDA_LIBS ${MLX_CUDNN_DLLS})+ list(APPEND MLX_CUDNN_LIBS ${MLX_CUDNN_DLLS})endif()- if(MLX_CUDA_LIBS)- install(FILES ${MLX_CUDA_LIBS}- DESTINATION ${OLLAMA_INSTALL_DIR}+ if(MLX_CUDNN_LIBS)+ list(REMOVE_DUPLICATES MLX_CUDNN_LIBS)+ install(FILES ${MLX_CUDNN_LIBS}+ DESTINATION ${OLLAMA_MLX_CUDNN_INSTALL_DIR}COMPONENT MLX_VENDOR)endif()endif()++if(OLLAMA_MLX_OVERLAY_PACKAGE)+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/README_MLX.txt"+ "Extract this MLX package to the same location as the Ollama base package.\n")+ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/README_MLX.txt"+ DESTINATION .+ COMPONENT MLX)+endif()diff --git a/mlx/dynamic.go b/mlx/dynamic.goindex 990c0bf3f..101162b04 100644--- a/mlx/dynamic.go+++ b/mlx/dynamic.go@@ -7,10 +7,10 @@ import "C"import ("fmt"- "io/fs""log/slog""os""path/filepath"+ "regexp""runtime""sort""strconv"@@ -46,16 +46,27 @@ func LoadedLibraryPath() (string, error) {return "", fmt.Errorf("MLX dynamic library not loaded")}-// tryLoadFromDir searches a directory for the mlxc shared library and loads it.-func tryLoadFromDir(dir string) bool {+// mlxLibraryCandidates returns the mlxc shared library in dir, if present.+func mlxLibraryCandidates(dir string) []string {// On Windows, MSVC produces mlxc.dll (no lib prefix)// On Unix, it's libmlxc.so or libmlxc.dylib- pattern := "libmlxc.*"- if runtime.GOOS == "windows" {- pattern = "mlxc.*"+ name := "libmlxc.so"+ switch runtime.GOOS {+ case "windows":+ name = "mlxc.dll"+ case "darwin":+ name = "libmlxc.dylib"+ }+ if info, err := os.Stat(filepath.Join(dir, name)); err != nil || !info.Mode().IsRegular() {+ return nil}- matches, err := fs.Glob(os.DirFS(dir), pattern)- if err != nil || len(matches) == 0 {+ return []string{name}+}++// tryLoadFromDir searches a directory for the mlxc shared library and loads it.+func tryLoadFromDir(dir string) bool {+ matches := mlxLibraryCandidates(dir)+ if len(matches) == 0 {return false}for _, match := range matches {@@ -167,24 +178,100 @@ func repoBuildDirs() []string {return dirs}-// prependLibraryPath prepends dir to the platform's dynamic library search-// path so the linker finds colocated libmlx before any stale copies.-// Called once after successful library load.-func prependLibraryPath(dir string) {- var envVar string+func libraryPathEnvVar() string {switch runtime.GOOS {+ case "windows":+ return "PATH"case "darwin":- envVar = "DYLD_LIBRARY_PATH"+ return "DYLD_LIBRARY_PATH"case "linux":- envVar = "LD_LIBRARY_PATH"+ return "LD_LIBRARY_PATH"default:- return+ return ""+ }+}++// prependLibraryPaths prepends dirs to the platform's dynamic library search+// path so MLX and any shared sibling backend dependencies resolve before stale+// system copies. It returns a restore function for failed speculative loads.+func prependLibraryPaths(dirs ...string) func() {+ envVar := libraryPathEnvVar()+ if envVar == "" {+ return func() {}+ }+ oldValue, hadValue := os.LookupEnv(envVar)++ var paths []string+ seen := map[string]bool{}+ add := func(path string) {+ if path == "" {+ return+ }+ path = filepath.Clean(path)+ if !seen[path] {+ seen[path] = true+ paths = append(paths, path)+ }+ }+ for _, dir := range dirs {+ add(dir)}if existing := os.Getenv(envVar); existing != "" {- os.Setenv(envVar, dir+string(filepath.ListSeparator)+existing)+ for _, dir := range filepath.SplitList(existing) {+ add(dir)+ }+ }+ if len(paths) == 0 {+ return func() {}+ }+ os.Setenv(envVar, strings.Join(paths, string(filepath.ListSeparator)))+ return func() {+ if hadValue {+ os.Setenv(envVar, oldValue)+ } else {+ os.Unsetenv(envVar)+ }+ }+}++func mlxLibraryPathsForDir(dir string) []string {+ paths := []string{dir}+ if depDir, ok := dependencyDir(dir); ok {+ paths = append(paths, depDir)+ }+ return paths+}++func tryLoadFromMLXDir(dir string) bool {+ restore := prependLibraryPaths(mlxLibraryPathsForDir(dir)...)+ if tryLoadFromDir(dir) {+ return true+ }+ restore()+ return false+}++func prependLoadedLibraryPath() {+ if initLoadedPath == "" {+ return+ }+ dir := filepath.Dir(initLoadedPath)+ if strings.HasPrefix(filepath.Base(dir), "mlx_") {+ prependLibraryPaths(mlxLibraryPathsForDir(dir)...)+ setBundledCUDAToolkitPath(dir)} else {- os.Setenv(envVar, dir)+ prependLibraryPaths(dir)+ }+}++func setBundledCUDAToolkitPath(dir string) {+ cudaPath, ok := cudaToolkitDirForMLXDir(dir)+ if !ok {+ return}+ os.Setenv("CUDA_PATH", cudaPath)+ os.Setenv("CUDA_HOME", cudaPath)+ slog.Debug("MLX CUDA headers", "CUDA_PATH", cudaPath)}func init() {@@ -208,13 +295,13 @@ func init() {return}- prependLibraryPath(filepath.Dir(initLoadedPath))+ prependLoadedLibraryPath()}func findMLXLibrary(forcedVariant string) bool {for _, root := range libOllamaRoots() {if forcedVariant != "" {- if tryLoadFromDir(filepath.Join(root, forcedVariant)) {+ if tryLoadFromMLXDir(filepath.Join(root, forcedVariant)) {return true}} else {@@ -229,6 +316,10 @@ func findMLXLibrary(forcedVariant string) bool {return false}+// mlxVariantPattern matches the variant directory names we ship, e.g.+// mlx_metal_v4, mlx_cuda_v13.+var mlxVariantPattern = regexp.MustCompile(`^mlx_(metal|cuda)_v\d+$`)+// tryLoadFromMLXSubdirs globs for mlx_* subdirs within dir, filters out// incompatible variants, tries the remainder in reverse sorted order (so// higher-versioned variants are preferred), and returns true on first@@ -241,11 +332,15 @@ func tryLoadFromMLXSubdirs(dir string) bool {// Reverse sort: mlx_metal_v4 before mlx_metal_v3, mlx_cuda_v13 before v12sort.Sort(sort.Reverse(sort.StringSlice(mlxDirs)))for _, mlxDir := range mlxDirs {+ if !mlxVariantPattern.MatchString(filepath.Base(mlxDir)) {+ slog.Debug("skipping unrecognized MLX variant", "dir", mlxDir)+ continue+ }if !isCompatibleMLXVariant(filepath.Base(mlxDir)) {slog.Debug("skipping incompatible MLX variant", "dir", mlxDir)continue}- if tryLoadFromDir(mlxDir) {+ if tryLoadFromMLXDir(mlxDir) {return true}}diff --git a/mlx/path.go b/mlx/path.gonew file mode 100644index 000000000..403aa2f12--- /dev/null+++ b/mlx/path.go@@ -0,0 +1,81 @@+package mlx++import (+ "os"+ "path/filepath"+ "strings"+)++// LibraryPathsForLoadedLibrary returns the runtime library search paths needed+// for the selected MLX dynamic library.+func LibraryPathsForLoadedLibrary(libraryPath string) []string {+ return libraryPathsForLibrary(libraryPath)+}++// CUDAToolkitDirForLoadedLibrary returns the selected MLX CUDA variant+// directory when it carries bundled CUDA headers for MLX JIT compilation.+func CUDAToolkitDirForLoadedLibrary(libraryPath string) (string, bool) {+ return cudaToolkitDirForLibrary(libraryPath)+}++func libraryPathsForLibrary(libraryPath string) []string {+ var paths []string+ seen := map[string]bool{}+ add := func(path string) {+ if path == "" {+ return+ }+ path = filepath.Clean(path)+ if !seen[path] {+ seen[path] = true+ paths = append(paths, path)+ }+ }++ dir := filepath.Dir(libraryPath)+ add(dir)+ if strings.HasPrefix(filepath.Base(dir), "mlx_") {+ if depDir, ok := dependencyDir(dir); ok {+ add(depDir)+ }+ }++ return paths+}++// dependencyDir maps an MLX CUDA variant directory to its shared dependency+// directory. Variants like mlx_cuda_v12 automatically use cuda_v12 when+// present; non-CUDA variants do not have a sibling dependency payload.+func dependencyDir(mlxDir string) (string, bool) {+ name := filepath.Base(mlxDir)+ cudaName, ok := strings.CutPrefix(name, "mlx_cuda_")+ if !ok || cudaName == "" {+ return "", false+ }+ depName := "cuda_" + cudaName++ depDir := filepath.Join(filepath.Dir(mlxDir), depName)+ if !dirExists(depDir) {+ return "", false+ }+ return depDir, true+}++func cudaToolkitDirForLibrary(libraryPath string) (string, bool) {+ return cudaToolkitDirForMLXDir(filepath.Dir(libraryPath))+}++func cudaToolkitDirForMLXDir(mlxDir string) (string, bool) {+ if !strings.HasPrefix(filepath.Base(mlxDir), "mlx_cuda_") {+ return "", false+ }+ if !dirExists(filepath.Join(mlxDir, "include")) {+ return "", false+ }+ return filepath.Clean(mlxDir), true+}++func dirExists(path string) bool {+ info, err := os.Stat(path)+ return err == nil && info.IsDir()+}diff --git a/mlx/path_test.go b/mlx/path_test.gonew file mode 100644index 000000000..61c6c9e8b--- /dev/null+++ b/mlx/path_test.go@@ -0,0 +1,128 @@+package mlx++import (+ "os"+ "path/filepath"+ "reflect"+ "testing"+)++func TestLibraryPathsForLoadedLibraryAddsOnlySelectedMLXDependency(t *testing.T) {+ root := t.TempDir()+ for _, dir := range []string{+ "mlx_cuda_v13",+ "cuda_v13",+ "mlx_cuda_v12",+ "cuda_v12",+ } {+ if err := os.Mkdir(filepath.Join(root, dir), 0o755); err != nil {+ t.Fatal(err)+ }+ }++ got := LibraryPathsForLoadedLibrary(filepath.Join(root, "mlx_cuda_v13", "mlxc.dll"))+ want := []string{+ filepath.Join(root, "mlx_cuda_v13"),+ filepath.Join(root, "cuda_v13"),+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("LibraryPathsForLoadedLibrary() = %#v, want %#v", got, want)+ }+}++func TestLibraryPathsForLoadedLibraryAddsStandaloneDir(t *testing.T) {+ root := t.TempDir()+ libDir := filepath.Join(root, "standalone")+ got := LibraryPathsForLoadedLibrary(filepath.Join(libDir, "libmlxc.so"))+ want := []string{libDir}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("LibraryPathsForLoadedLibrary() = %#v, want %#v", got, want)+ }+}++func TestDependencyDirRequiresExistingSibling(t *testing.T) {+ root := t.TempDir()+ mlxDir := filepath.Join(root, "mlx_cuda_v13")+ if err := os.Mkdir(mlxDir, 0o755); err != nil {+ t.Fatal(err)+ }++ if depDir, ok := dependencyDir(mlxDir); ok {+ t.Fatalf("dependencyDir() = %q, true; want missing dependency", depDir)+ }++ cudaDir := filepath.Join(root, "cuda_v13")+ if err := os.Mkdir(cudaDir, 0o755); err != nil {+ t.Fatal(err)+ }+ if depDir, ok := dependencyDir(mlxDir); !ok || depDir != cudaDir {+ t.Fatalf("dependencyDir() = %q, %v; want %q, true", depDir, ok, cudaDir)+ }+}++func TestDependencyDirRejectsNonCUDAVariant(t *testing.T) {+ root := t.TempDir()+ for _, dir := range []string{"mlx_metal_v3", "metal_v3"} {+ if err := os.Mkdir(filepath.Join(root, dir), 0o755); err != nil {+ t.Fatal(err)+ }+ }++ if depDir, ok := dependencyDir(filepath.Join(root, "mlx_metal_v3")); ok {+ t.Fatalf("dependencyDir() = %q, true; want non-CUDA variant rejected", depDir)+ }+}++func TestCUDAToolkitDirForLoadedLibraryRequiresMLXCUDAInclude(t *testing.T) {+ root := t.TempDir()+ for _, dir := range []string{+ "mlx_cuda_v13",+ "mlx_test_v1",+ } {+ if err := os.Mkdir(filepath.Join(root, dir), 0o755); err != nil {+ t.Fatal(err)+ }+ }+ libraryPath := filepath.Join(root, "mlx_cuda_v13", "mlxc.dll")++ if cudaDir, ok := CUDAToolkitDirForLoadedLibrary(libraryPath); ok {+ t.Fatalf("CUDAToolkitDirForLoadedLibrary() = %q, true; want no CUDA headers", cudaDir)+ }++ want := filepath.Join(root, "mlx_cuda_v13")+ if err := os.Mkdir(filepath.Join(want, "include"), 0o755); err != nil {+ t.Fatal(err)+ }+ if cudaDir, ok := CUDAToolkitDirForLoadedLibrary(libraryPath); !ok || cudaDir != want {+ t.Fatalf("CUDAToolkitDirForLoadedLibrary() = %q, %v; want %q, true", cudaDir, ok, want)+ }+}++func TestCUDAToolkitDirForLoadedLibraryUsesSelectedCUDAName(t *testing.T) {+ root := t.TempDir()+ for _, dir := range []string{+ filepath.Join(root, "mlx_cuda_v12", "include"),+ filepath.Join(root, "mlx_cuda_v13", "include"),+ } {+ if err := os.MkdirAll(dir, 0o755); err != nil {+ t.Fatal(err)+ }+ }++ want := filepath.Join(root, "mlx_cuda_v12")+ if cudaDir, ok := CUDAToolkitDirForLoadedLibrary(filepath.Join(want, "mlxc.dll")); !ok || cudaDir != want {+ t.Fatalf("CUDAToolkitDirForLoadedLibrary() = %q, %v; want %q, true", cudaDir, ok, want)+ }+}++func TestCUDAToolkitDirForMLXDirRejectsNonCUDAVariants(t *testing.T) {+ root := t.TempDir()+ otherDir := filepath.Join(root, "mlx_test_v1")+ if err := os.MkdirAll(filepath.Join(otherDir, "include"), 0o755); err != nil {+ t.Fatal(err)+ }++ if cudaDir, ok := cudaToolkitDirForMLXDir(otherDir); ok {+ t.Fatalf("cudaToolkitDirForMLXDir() = %q, true; want rejected non-CUDA variant", cudaDir)+ }+}diff --git a/mlxrunner/client.go b/mlxrunner/client.goindex 6c92727b8..649473622 100644--- a/mlxrunner/client.go+++ b/mlxrunner/client.go@@ -438,6 +438,10 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D// Spawn subprocess: ollama runner --model <name> --port <port>cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port))cmd.Env = os.Environ()+ loadedLibraryPath, err := mlx.LoadedLibraryPath()+ if err != nil {+ return nil, fmt.Errorf("unable to determine loaded MLX library path: %w", err)+ }// Keep Metal weights resident between requests until MLX fixes idle eviction.if runtime.GOOS == "darwin" {@@ -457,16 +461,8 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D}if libPathEnvVar != "" {- libraryPaths := []string{ml.LibOllamaPath}- if mlxDirs, err := filepath.Glob(filepath.Join(ml.LibOllamaPath, "mlx_*")); err == nil {- libraryPaths = append(libraryPaths, mlxDirs...)- }-- if existingPath, ok := os.LookupEnv(libPathEnvVar); ok {- libraryPaths = append(libraryPaths, filepath.SplitList(existingPath)...)- }-- pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))+ libraryPaths := mlx.LibraryPathsForLoadedLibrary(loadedLibraryPath)+ pathEnvVal := libraryPathEnv(libraryPaths, os.Getenv(libPathEnvVar))found := falsefor i := range cmd.Env {@@ -490,15 +486,10 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D// MLX resolves headers via $CUDA_PATH/include/*.h (and checks CUDA_HOME first).// Always use bundled headers to avoid version mismatches with any// system-installed CUDA toolkit.- if mlxDirs, err := filepath.Glob(filepath.Join(ml.LibOllamaPath, "mlx_cuda_*")); err == nil {- for _, d := range mlxDirs {- if _, err := os.Stat(filepath.Join(d, "include")); err == nil {- setEnv(cmd, "CUDA_PATH", d)- setEnv(cmd, "CUDA_HOME", d)- slog.Debug("mlx subprocess CUDA headers", "CUDA_PATH", d)- break- }- }+ if cudaPath, ok := mlx.CUDAToolkitDirForLoadedLibrary(loadedLibraryPath); ok {+ setEnv(cmd, "CUDA_PATH", cudaPath)+ setEnv(cmd, "CUDA_HOME", cudaPath)+ slog.Debug("mlx subprocess CUDA headers", "CUDA_PATH", cudaPath)}status := llm.NewStatusWriter(os.Stderr)@@ -653,3 +644,31 @@ func setEnv(cmd *exec.Cmd, key, value string) {}cmd.Env = append(cmd.Env, entry)}++func libraryPathEnv(preferred []string, existing string) string {+ var paths []string+ seen := map[string]bool{}+ add := func(path string) {+ if path == "" {+ return+ }+ path = filepath.Clean(path)+ key := path+ if runtime.GOOS == "windows" {+ key = strings.ToUpper(key)+ }+ if !seen[key] {+ seen[key] = true+ paths = append(paths, path)+ }+ }++ for _, path := range preferred {+ add(path)+ }+ for _, path := range filepath.SplitList(existing) {+ add(path)+ }++ return strings.Join(paths, string(filepath.ListSeparator))+}diff --git a/scripts/build_linux.sh b/scripts/build_linux.shindex b4485e75a..ac85b069d 100755--- a/scripts/build_linux.sh+++ b/scripts/build_linux.sh@@ -40,14 +40,6 @@ docker buildx build \-f Dockerfile \.-# Run deduplication for each platform output directory-if echo $PLATFORM | grep "," > /dev/null ; then- $(dirname $0)/deduplicate_cuda_libs.sh "./dist/linux_amd64"- $(dirname $0)/deduplicate_cuda_libs.sh "./dist/linux_arm64"-elif echo $PLATFORM | grep "amd64\|arm64" > /dev/null ; then- $(dirname $0)/deduplicate_cuda_libs.sh "./dist"-fi-# buildx behavior changes for single vs. multiplatformecho "Compressing linux tar bundles..."if echo $PLATFORM | grep "," > /dev/null ; thendiff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1index 0b3c3b3ba..5dbd23bf8 100644--- a/scripts/build_windows.ps1+++ b/scripts/build_windows.ps1@@ -840,7 +840,8 @@ function mlxCuda13 {$cudaFlags += "-DCMAKE_CUDA_FLAGS=$env:OLLAMA_CMAKE_CUDA_FLAGS"}$cudaToolsetArgs = cudaCMakeArgs $cuda- $configureArgs = @("-S", ".", "-B", "build\mlx_cuda_v$cudaMajorVer", "-DOLLAMA_MLX_BACKENDS=cuda_v$cudaMajorVer") + $cudaToolsetArgs + $cudaFlags + @("-DOLLAMA_PAYLOAD_INSTALL_PREFIX=$script:DIST_DIR", "--install-prefix", "$script:DIST_DIR")+ $mlxDistDir = "${script:SRC_DIR}\dist\windows-amd64-mlx"+ $configureArgs = @("-S", ".", "-B", "build\mlx_cuda_v$cudaMajorVer", "-DCMAKE_BUILD_TYPE=Release", "-DOLLAMA_MLX_BACKENDS=cuda_v$cudaMajorVer") + $cudaToolsetArgs + $cudaFlags + @("-DOLLAMA_PAYLOAD_INSTALL_PREFIX=$script:DIST_DIR", "-DOLLAMA_MLX_PAYLOAD_INSTALL_PREFIX=$mlxDistDir", "--install-prefix", "$script:DIST_DIR")& cmake @configureArgsif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}$buildArgs = @("--build", "build\mlx_cuda_v$cudaMajorVer", "--target", "ollama-mlx-cuda_v$cudaMajorVer", "--config", "Release", "--parallel", "$script:JOBS")@@ -1245,8 +1246,8 @@ function zip {$jobs += newDependencyAuditJob "${distDir}\windows-amd64-rocm" "windows-amd64-rocm" "${distDir}\dependency-audit-windows-amd64-rocm.txt" $amd64Dir}- # Stage MLX into its own directory for independent compression- if (stageComponents $amd64Dir "${distDir}\windows-amd64-mlx" "mlx_*" "MLX") {+ # CMake installs the MLX overlay into its own staging directory.+ if (Test-Path -Path "${distDir}\windows-amd64-mlx\lib\ollama") {Write-Output "Generating ${distDir}\ollama-windows-amd64-mlx.zip"$jobs += newZipJob "${distDir}\windows-amd64-mlx" "${distDir}\ollama-windows-amd64-mlx.zip"$jobs += newDependencyAuditJob "${distDir}\windows-amd64-mlx" "windows-amd64-mlx" "${distDir}\dependency-audit-windows-amd64-mlx.txt" $amd64Dir@@ -1287,7 +1288,6 @@ function zip {} finally {# Always restore staged components back into the main treerestoreComponents $amd64Dir "${distDir}\windows-amd64-rocm"- restoreComponents $amd64Dir "${distDir}\windows-amd64-mlx"}}diff --git a/scripts/deduplicate_cuda_libs.sh b/scripts/deduplicate_cuda_libs.shdeleted file mode 100755index f8d13cd3d..000000000--- a/scripts/deduplicate_cuda_libs.sh+++ /dev/null@@ -1,60 +0,0 @@-#!/bin/sh-#-# Deduplicate CUDA libraries across mlx_* and cuda_* directories-# This script finds identical .so* files in mlx_cuda_* directories that exist-# in corresponding cuda_* directories and replaces them with symlinks.-#--set -eu--if [ $# -eq 0 ]; then- echo "ERROR: No directory specified" >&2- echo "Usage: $0 <base_directory>" >&2- exit 1-fi--base_dir="$1"--if [ ! -d "${base_dir}" ]; then- echo "ERROR: Directory ${base_dir} does not exist" >&2- exit 1-fi--echo "Deduplicating CUDA libraries in ${base_dir}..."--# Find all mlx_cuda_* directories-for mlx_dir in "${base_dir}"/lib/ollama/mlx_cuda_*; do- [ -d "${mlx_dir}" ] || continue-- # Extract CUDA version (e.g., v12, v13)- cuda_version=$(basename "${mlx_dir}" | sed 's/mlx_cuda_//')- cuda_dir="${base_dir}/lib/ollama/cuda_${cuda_version}"-- # Skip if corresponding cuda_* directory doesn't exist- [ -d "${cuda_dir}" ] || continue-- echo " Checking ${mlx_dir} against ${cuda_dir}..."-- # Find all .so* files in mlx directory- find "${mlx_dir}" -type f -name "*.so*" | while read mlx_file; do- filename=$(basename "${mlx_file}")- cuda_file="${cuda_dir}/${filename}"-- # Skip if file doesn't exist in cuda directory- [ -f "${cuda_file}" ] || continue-- # Compare checksums- mlx_sum=$(sha256sum "${mlx_file}" | awk '{print $1}')- cuda_sum=$(sha256sum "${cuda_file}" | awk '{print $1}')-- if [ "${mlx_sum}" = "${cuda_sum}" ]; then- echo " Deduplicating ${filename}"- # Calculate relative path from mlx_dir to cuda_dir- rel_path="../cuda_${cuda_version}/${filename}"- rm -f "${mlx_file}"- ln -s "${rel_path}" "${mlx_file}"- fi- done-done--echo "Deduplication complete"