Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

mlx: deduplicate CUDA runtime payloads (#17956)

Daniel Hiltgen2cd63f7a1503
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 8517dcd1e..58acb9443 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -659,9 +659,6 @@ jobs:
build-args: ${{ matrix.build-args }}
outputs: type=local,dest=dist/${{ matrix.os }}-${{ matrix.arch }}
cache-from: ${{ matrix.cache-from }}
- - name: Deduplicate CUDA libraries
- run: |
- ./scripts/deduplicate_cuda_libs.sh dist/${{ matrix.os }}-${{ matrix.arch }}
- name: Verify Linux build payloads
shell: bash
run: |
@@ -673,6 +670,24 @@ jobs:
do
[ -f "$payload" ] || { echo "missing $payload"; exit 1; }
done
+ if [ "${{ matrix.arch }}" = "amd64" ] && [ "${{ matrix.suffix }}" != "-rocm" ]; then
+ for payload in \
+ "$base/lib/ollama/mlx_cuda_v13/libopenblas.so.0" \
+ "$base/lib/ollama/mlx_cuda_v13/libgfortran.so.5" \
+ "$base/lib/ollama/mlx_cuda_v13/libquadmath.so.0"
+ do
+ [ -f "$payload" ] || { echo "missing $payload"; exit 1; }
+ done
+ mlx_deps="$(ldd "$base/lib/ollama/mlx_cuda_v13/libmlxc.so")"
+ echo "$mlx_deps"
+ for dependency in libgfortran.so.5 libquadmath.so.0
+ do
+ echo "$mlx_deps" | grep -E "^[[:space:]]*$dependency => .*mlx_cuda_v13/" >/dev/null || {
+ echo "$dependency does not resolve from the MLX payload"
+ exit 1
+ }
+ done
+ fi
- run: |
for COMPONENT in bin/* lib/ollama/*; do
case "$COMPONENT" in
diff --git a/.github/workflows/test-llamacpp-update.yaml b/.github/workflows/test-llamacpp-update.yaml
index 542c766e2..81ba9db7b 100644
--- a/.github/workflows/test-llamacpp-update.yaml
+++ b/.github/workflows/test-llamacpp-update.yaml
@@ -218,7 +218,6 @@ jobs:
copy_payload cuda_jetpack6
fi
- ./scripts/deduplicate_cuda_libs.sh "${dest}"
- name: Verify Linux build payloads
shell: bash
run: |
diff --git a/cmake/local.cmake b/cmake/local.cmake
index b2dad3adb..3c7d18f2b 100644
--- a/cmake/local.cmake
+++ b/cmake/local.cmake
@@ -12,6 +12,20 @@ set(_ollama_mlx_backends_doc "Semicolon-separated MLX backends to build: cuda_v1
set(OLLAMA_VERSION "0.0.0" CACHE STRING "Ollama version embedded in the local Go binary")
set(OLLAMA_PAYLOAD_INSTALL_PREFIX "${CMAKE_BINARY_DIR}" CACHE PATH
"Build-time staging prefix for nested Ollama native payloads")
+set(OLLAMA_MLX_PAYLOAD_INSTALL_PREFIX "" CACHE PATH
+ "Optional separate staging prefix for the MLX overlay payload")
+
+get_filename_component(_ollama_mlx_base_install_prefix
+ "${OLLAMA_PAYLOAD_INSTALL_PREFIX}" ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}")
+set(_ollama_mlx_payload_install_prefix "${_ollama_mlx_base_install_prefix}")
+set(_ollama_mlx_overlay_package OFF)
+if(OLLAMA_MLX_PAYLOAD_INSTALL_PREFIX)
+ get_filename_component(_ollama_mlx_payload_install_prefix
+ "${OLLAMA_MLX_PAYLOAD_INSTALL_PREFIX}" ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}")
+ if(NOT "${_ollama_mlx_payload_install_prefix}" STREQUAL "${_ollama_mlx_base_install_prefix}")
+ set(_ollama_mlx_overlay_package ON)
+ endif()
+endif()
string(REGEX REPLACE "^v" "" OLLAMA_VERSION "${OLLAMA_VERSION}")
@@ -102,6 +116,12 @@ else()
set(OLLAMA_MLX_BACKENDS "${OLLAMA_MLX_BACKENDS}" CACHE STRING "${_ollama_mlx_backends_doc}")
endif()
+if(OLLAMA_MLX_BACKENDS AND _ollama_mlx_overlay_package)
+ add_custom_target(ollama-mlx-overlay-clean
+ COMMAND ${CMAKE_COMMAND} -E rm -rf "${_ollama_mlx_payload_install_prefix}"
+ COMMENT "Cleaning MLX overlay staging directory")
+endif()
+
if(NOT OLLAMA_HAVE_LLAMA_SERVER)
if(OLLAMA_LLAMA_BACKENDS)
message(FATAL_ERROR "llama/server is required when OLLAMA_LLAMA_BACKENDS is set")
@@ -528,13 +548,14 @@ function(ollama_add_mlx_build name)
ollama_collect_cache_args_with_prefix("MLX_" _mlx_cache_args)
set(_cmake_args
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
- -DCMAKE_INSTALL_PREFIX=${OLLAMA_PAYLOAD_INSTALL_PREFIX}
+ -DCMAKE_INSTALL_PREFIX=${_ollama_mlx_base_install_prefix}
-DOLLAMA_LIB_DIR:STRING=${OLLAMA_LIB_DIR}
-DOLLAMA_RUNNER_DIR=${ARG_RUNNER_DIR}
-DOLLAMA_SOURCE_DIR=${CMAKE_SOURCE_DIR}
-DFETCHCONTENT_SOURCE_DIR_MLX=${OLLAMA_MLX_SOURCE_DIR}
-DFETCHCONTENT_SOURCE_DIR_MLX-C=${OLLAMA_MLX_C_SOURCE_DIR}
-DOLLAMA_MLX_GENERATE_WRAPPERS=OFF
+ -DOLLAMA_MLX_OVERLAY_PACKAGE=${_ollama_mlx_overlay_package}
${ARG_CMAKE_ARGS}
${_mlx_cache_args}
)
@@ -575,6 +596,26 @@ function(ollama_add_mlx_build name)
${_cmake_args})
endif()
+ set(_install_command)
+ list(APPEND _install_command
+ ${CMAKE_COMMAND} --install <BINARY_DIR>
+ ${OLLAMA_NATIVE_CONFIG_ARG}
+ --prefix "${_ollama_mlx_payload_install_prefix}"
+ --component MLX
+ COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>
+ ${OLLAMA_NATIVE_CONFIG_ARG}
+ --prefix "${_ollama_mlx_payload_install_prefix}"
+ --component MLX_VENDOR
+ COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>
+ ${OLLAMA_NATIVE_CONFIG_ARG}
+ --prefix "${_ollama_mlx_base_install_prefix}"
+ --component MLX_SHARED)
+
+ set(_mlx_dependencies ollama-mlx-sources)
+ if(_ollama_mlx_overlay_package)
+ list(APPEND _mlx_dependencies ollama-mlx-overlay-clean)
+ endif()
+
ExternalProject_Add(ollama-mlx-${name}
SOURCE_DIR ${CMAKE_SOURCE_DIR}/cmake/mlx
BINARY_DIR ${_build_dir}
@@ -584,13 +625,8 @@ function(ollama_add_mlx_build name)
${OLLAMA_NATIVE_BUILD_TARGET_ARG} mlx
${OLLAMA_NATIVE_BUILD_TARGET_ARG} mlxc
${OLLAMA_NATIVE_BUILD_TARGET_ARG} ollama_xgrammar
- INSTALL_COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>
- ${OLLAMA_NATIVE_CONFIG_ARG}
- --component MLX
- COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>
- ${OLLAMA_NATIVE_CONFIG_ARG}
- --component MLX_VENDOR
- DEPENDS ollama-mlx-sources
+ INSTALL_COMMAND ${_install_command}
+ DEPENDS ${_mlx_dependencies}
LIST_SEPARATOR |
BUILD_ALWAYS TRUE
${OLLAMA_NATIVE_EXTERNAL_OPTIONS}
diff --git a/cmake/mlx/CMakeLists.txt b/cmake/mlx/CMakeLists.txt
index d1f880158..ade40df92 100644
--- a/cmake/mlx/CMakeLists.txt
+++ b/cmake/mlx/CMakeLists.txt
@@ -19,16 +19,6 @@ set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS ON)
-if(APPLE)
- set(CMAKE_BUILD_RPATH "@loader_path")
- set(CMAKE_INSTALL_RPATH "@loader_path")
- set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
-elseif(UNIX)
- set(CMAKE_BUILD_RPATH "$ORIGIN")
- set(CMAKE_INSTALL_RPATH "$ORIGIN")
- set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
-endif()
-
if(NOT DEFINED OLLAMA_SOURCE_DIR OR "${OLLAMA_SOURCE_DIR}" STREQUAL "")
get_filename_component(OLLAMA_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE)
endif()
@@ -39,6 +29,42 @@ set(OLLAMA_LIB_DIR "lib/ollama" CACHE STRING "Install destination for Ollama run
set(OLLAMA_RUNNER_DIR "" CACHE STRING "Ollama runtime payload subdirectory")
set(OLLAMA_BUILD_DIR ${CMAKE_BINARY_DIR}/lib/ollama)
set(OLLAMA_INSTALL_DIR ${OLLAMA_LIB_DIR}/${OLLAMA_RUNNER_DIR})
+set(OLLAMA_DEPENDENCY_INSTALL_DIR "")
+set(OLLAMA_DEPENDENCY_RUNNER_DIR "")
+if(OLLAMA_RUNNER_DIR MATCHES "^mlx_(cuda.*)$")
+ set(OLLAMA_DEPENDENCY_RUNNER_DIR "${CMAKE_MATCH_1}")
+ set(OLLAMA_DEPENDENCY_INSTALL_DIR "${OLLAMA_LIB_DIR}/${OLLAMA_DEPENDENCY_RUNNER_DIR}")
+endif()
+
+set(OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS "${OLLAMA_INSTALL_DIR}/include")
+set(OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR "${OLLAMA_INSTALL_DIR}")
+set(OLLAMA_MLX_PRIVATE_CUDA_INSTALL_DIR "${OLLAMA_INSTALL_DIR}")
+set(OLLAMA_MLX_CUDNN_INSTALL_DIR "${OLLAMA_INSTALL_DIR}")
+set(OLLAMA_MLX_RPATH)
+
+if(APPLE)
+ set(OLLAMA_MLX_RPATH "@loader_path")
+elseif(WIN32)
+ if(OLLAMA_DEPENDENCY_INSTALL_DIR)
+ set(OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR "${OLLAMA_DEPENDENCY_INSTALL_DIR}")
+ set(OLLAMA_MLX_PRIVATE_CUDA_INSTALL_DIR "${OLLAMA_DEPENDENCY_INSTALL_DIR}")
+ set(MLX_CUDA_BIN_DIR "../${OLLAMA_DEPENDENCY_RUNNER_DIR}" CACHE STRING "" FORCE)
+ set(MLX_CUDNN_BIN_DIR "." CACHE STRING "" FORCE)
+ endif()
+elseif(UNIX)
+ set(OLLAMA_MLX_RPATH "$ORIGIN")
+ list(APPEND OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS "${OLLAMA_LIB_DIR}/include")
+ if(OLLAMA_DEPENDENCY_INSTALL_DIR)
+ set(OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR "${OLLAMA_DEPENDENCY_INSTALL_DIR}")
+ list(APPEND OLLAMA_MLX_RPATH "$ORIGIN/../${OLLAMA_DEPENDENCY_RUNNER_DIR}")
+ endif()
+endif()
+
+if(OLLAMA_MLX_RPATH)
+ set(CMAKE_BUILD_RPATH ${OLLAMA_MLX_RPATH})
+ set(CMAKE_INSTALL_RPATH ${OLLAMA_MLX_RPATH})
+ set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
+endif()
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OLLAMA_BUILD_DIR})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${OLLAMA_BUILD_DIR})
@@ -52,6 +78,7 @@ if(MLX_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES)
endif()
option(OLLAMA_MLX_GENERATE_WRAPPERS "Regenerate MLX Go wrappers" OFF)
+option(OLLAMA_MLX_OVERLAY_PACKAGE "Install the MLX payload as an overlay package" OFF)
message(STATUS "Setting up MLX (this takes a while...)")
foreach(_cudnn_var CUDNN_INCLUDE_PATH CUDNN_LIBRARY_PATH)
@@ -61,6 +88,20 @@ foreach(_cudnn_var CUDNN_INCLUDE_PATH CUDNN_LIBRARY_PATH)
endforeach()
add_subdirectory(${OLLAMA_SOURCE_DIR}/mlx ${CMAKE_BINARY_DIR}/mlx)
+if(UNIX AND NOT APPLE)
+ foreach(_mlx_target mlx mlxc)
+ if(TARGET ${_mlx_target})
+ set_target_properties(${_mlx_target} PROPERTIES
+ BUILD_RPATH "${OLLAMA_MLX_RPATH}"
+ INSTALL_RPATH "${OLLAMA_MLX_RPATH}"
+ BUILD_WITH_INSTALL_RPATH ON)
+ # DT_RPATH is inherited by transitive dependencies such as
+ # OpenBLAS -> libgfortran -> libquadmath; DT_RUNPATH is not.
+ target_link_options(${_mlx_target} PRIVATE LINKER:--disable-new-dtags)
+ endif()
+ endforeach()
+endif()
+
include(FetchContent)
set(XGRAMMAR_VERSION v0.2.7)
FetchContent_Declare(xgrammar
@@ -93,7 +134,7 @@ add_subdirectory(${OLLAMA_SOURCE_DIR}/mlxrunner/xgrammar/native ${CMAKE_BINARY_D
find_package(CUDAToolkit)
# Build list of directories for runtime dependency resolution.
-set(MLX_RUNTIME_DIRS ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_BIN_DIR}/x64 ${CUDAToolkit_LIBRARY_DIR})
+set(MLX_RUNTIME_SEARCH_DIRS ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_BIN_DIR}/x64 ${CUDAToolkit_LIBRARY_DIR})
# Add cuDNN bin paths for DLLs (Windows MLX CUDA builds).
# CUDNN_ROOT_DIR is the standard CMake variable for cuDNN location.
if(CUDNN_ROOT_DIR)
@@ -105,24 +146,62 @@ if(_cudnn_root)
file(TO_CMAKE_PATH "${_cudnn_root}" _cudnn_root)
# cuDNN 9.x has versioned subdirectories under bin/ (e.g., bin/13.0/).
file(GLOB CUDNN_BIN_SUBDIRS "${_cudnn_root}/bin/*")
- list(APPEND MLX_RUNTIME_DIRS ${CUDNN_BIN_SUBDIRS})
+ list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CUDNN_BIN_SUBDIRS})
endif()
# Add build output directory and MLX dependency build directories.
-list(APPEND MLX_RUNTIME_DIRS ${OLLAMA_BUILD_DIR})
+list(APPEND MLX_RUNTIME_SEARCH_DIRS ${OLLAMA_BUILD_DIR})
+if(UNIX AND NOT APPLE)
+ list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES})
+endif()
# OpenBLAS DLL location (pre-built zip extracts into openblas-src/bin/).
-list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/openblas-src/bin)
+list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CMAKE_BINARY_DIR}/_deps/openblas-src/bin)
# NCCL: on Linux, if real NCCL is found, cmake bundles libnccl.so via the
# regex below. If NCCL is not found, MLX links a static stub (OBJECT lib)
# so there is no runtime dependency. This path covers the stub build dir
# for windows so we include the DLL in our dependencies.
-list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/distributed/nccl/nccl_stub-prefix/src/nccl_stub-build/Release)
+list(APPEND MLX_RUNTIME_SEARCH_DIRS ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/distributed/nccl/nccl_stub-prefix/src/nccl_stub-build/Release)
# Non-link-time deps stay explicit; link-time deps derive from the mlx target.
-set(MLX_INCLUDE_REGEXES cublas cublasLt cudart cufft nvrtc nvrtc-builtins cudnn nccl cusolver cusparse nv[Jj]it[Ll]ink openblas gfortran)
+set(MLX_INCLUDE_REGEXES cublas cublasLt cudart cufft nvrtc nvrtc-builtins cudnn nccl cusolver cusparse nv[Jj]it[Ll]ink openblas gfortran quadmath)
# On Windows, also include dl.dll (dlfcn-win32 POSIX emulation layer).
if(WIN32)
list(APPEND MLX_INCLUDE_REGEXES "^dl\\.dll$")
endif()
+set(MLX_RUNTIME_DEPENDENCY_FILTERS
+ DIRECTORIES ${MLX_RUNTIME_SEARCH_DIRS}
+ PRE_INCLUDE_REGEXES ${MLX_INCLUDE_REGEXES}
+ PRE_EXCLUDE_REGEXES ".*")
+set(MLX_EXPLICIT_CUDA_RUNTIME_REGEXES)
+if(WIN32)
+ list(APPEND MLX_EXPLICIT_CUDA_RUNTIME_REGEXES
+ "[/\\\\]cublas[^/\\\\]*\\.dll$"
+ "[/\\\\]cudart[^/\\\\]*\\.dll$"
+ "[/\\\\]cudnn[^/\\\\]*\\.dll$"
+ "[/\\\\]cufft[^/\\\\]*\\.dll$"
+ "[/\\\\]cusolver[^/\\\\]*\\.dll$"
+ "[/\\\\]cusparse[^/\\\\]*\\.dll$"
+ "[/\\\\]nvfatbin[^/\\\\]*\\.dll$"
+ "[/\\\\][nN][vV][jJ][iI][tT][lL][iI][nN][kK][^/\\\\]*\\.dll$"
+ "[/\\\\]nvrtc[^/\\\\]*\\.dll$"
+ "[/\\\\]nvrtc-builtins[^/\\\\]*\\.dll$")
+elseif(UNIX AND NOT APPLE)
+ list(APPEND MLX_EXPLICIT_CUDA_RUNTIME_REGEXES
+ "[/\\\\]libcublas\\.so(\\..*)?$"
+ "[/\\\\]libcublasLt\\.so(\\..*)?$"
+ "[/\\\\]libcudart\\.so(\\..*)?$"
+ "[/\\\\]libcudnn[^/\\\\]*\\.so(\\..*)?$"
+ "[/\\\\]libcufft\\.so(\\..*)?$"
+ "[/\\\\]libcusolver\\.so(\\..*)?$"
+ "[/\\\\]libcusparse\\.so(\\..*)?$"
+ "[/\\\\]libnvfatbin\\.so(\\..*)?$"
+ "[/\\\\]libnvJitLink\\.so(\\..*)?$"
+ "[/\\\\]libnvrtc\\.so(\\..*)?$"
+ "[/\\\\]libnvrtc-builtins\\.so(\\..*)?$")
+endif()
+if(MLX_EXPLICIT_CUDA_RUNTIME_REGEXES)
+ list(APPEND MLX_RUNTIME_DEPENDENCY_FILTERS
+ POST_EXCLUDE_REGEXES ${MLX_EXPLICIT_CUDA_RUNTIME_REGEXES})
+endif()
# Keep mlx/mlxc targets separate from runtime dependencies so --strip only
# applies to the binaries we build, not vendor DLLs/libs.
@@ -198,9 +277,7 @@ install(CODE "
" COMPONENT MLX)
install(RUNTIME_DEPENDENCY_SET mlx_runtime_deps
- DIRECTORIES ${MLX_RUNTIME_DIRS}
- PRE_INCLUDE_REGEXES ${MLX_INCLUDE_REGEXES}
- PRE_EXCLUDE_REGEXES ".*"
+ ${MLX_RUNTIME_DEPENDENCY_FILTERS}
RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDOR
LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDOR
)
@@ -222,21 +299,6 @@ if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
COMPONENT MLX)
endif()
-# Install headers for NVRTC JIT compilation at runtime.
-# MLX's own install rules use the default component so they get skipped by
-# --component MLX. Headers are installed alongside libmlx in OLLAMA_INSTALL_DIR.
-#
-# Layout:
-# ${OLLAMA_INSTALL_DIR}/include/cccl/ - CCCL headers
-# ${OLLAMA_INSTALL_DIR}/include/{cute,cutlass}/ - CUTLASS/CUTE headers
-# ${OLLAMA_INSTALL_DIR}/include/ - CUDA runtime/core headers
-#
-# MLX's jit_module.cpp resolves JIT support headers from the backend-local
-# include directory. On Linux it also probes current_binary_dir().parent_path()
-# / "include", so we create a symlink from lib/ollama/include to the backend
-# include directory for archive packaging.
-# This will need refinement if we add multiple CUDA versions for MLX in the future.
-# CUDA runtime headers are found via CUDA_PATH env var (set by mlxrunner).
set(_mlx_jit_cccl_include_dir "")
if(CUDAToolkit_FOUND)
foreach(_dir ${CUDAToolkit_INCLUDE_DIRS})
@@ -252,28 +314,25 @@ endif()
if(_mlx_jit_cccl_include_dir)
foreach(_cccl_dir cuda nv cub thrust)
if(EXISTS "${_mlx_jit_cccl_include_dir}/${_cccl_dir}")
- install(DIRECTORY "${_mlx_jit_cccl_include_dir}/${_cccl_dir}"
- DESTINATION ${OLLAMA_INSTALL_DIR}/include/cccl
- COMPONENT MLX)
+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)
+ install(DIRECTORY "${_mlx_jit_cccl_include_dir}/${_cccl_dir}"
+ DESTINATION ${_dst}/cccl
+ COMPONENT MLX)
+ endforeach()
endif()
endforeach()
endif()
if(EXISTS ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cute)
- install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cute
- DESTINATION ${OLLAMA_INSTALL_DIR}/include
- COMPONENT MLX)
- install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cutlass
- DESTINATION ${OLLAMA_INSTALL_DIR}/include
- COMPONENT MLX)
+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)
+ install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cute
+ DESTINATION ${_dst}
+ COMPONENT MLX)
+ install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include/cutlass
+ DESTINATION ${_dst}
+ COMPONENT MLX)
+ endforeach()
endif()
-# Install CUDA runtime/core headers needed by MLX JIT kernels.
-# NVIDIA's NVRTC bundled-header model is CUDA Runtime + CCCL, not the entire
-# toolkit include tree. Keep CCCL coherent above, include CUTLASS/CUTE above,
-# and avoid shipping unrelated SDK headers such as NPP, CUPTI, cuRAND, NVML,
-# cuBLAS, cuSPARSE, and cuSOLVER.
-# The Go mlxrunner sets CUDA_PATH to OLLAMA_INSTALL_DIR so MLX finds them at
-# $CUDA_PATH/include via NVRTC --include-path.
if(CUDAToolkit_FOUND)
# CUDAToolkit_INCLUDE_DIRS may be a semicolon-separated list
# (e.g. ".../include;.../include/cccl"). Find the entry that
@@ -288,8 +347,6 @@ if(CUDAToolkit_FOUND)
if(NOT _cuda_inc)
message(WARNING "Could not find cuda_runtime_api.h in CUDAToolkit_INCLUDE_DIRS: ${CUDAToolkit_INCLUDE_DIRS}")
else()
- set(_dst "${OLLAMA_INSTALL_DIR}/include")
-
set(_mlx_jit_cuda_headers
builtin_types.h
channel_descriptor.h
@@ -367,28 +424,23 @@ if(CUDAToolkit_FOUND)
endif()
endforeach()
if(_mlx_jit_cuda_header_paths)
- install(FILES ${_mlx_jit_cuda_header_paths}
- DESTINATION ${_dst}
- COMPONENT MLX)
+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)
+ install(FILES ${_mlx_jit_cuda_header_paths}
+ DESTINATION ${_dst}
+ COMPONENT MLX)
+ endforeach()
endif()
foreach(_runtime_dir cooperative_groups crt)
if(EXISTS "${_cuda_inc}/${_runtime_dir}")
- install(DIRECTORY "${_cuda_inc}/${_runtime_dir}"
- DESTINATION ${_dst}
- COMPONENT MLX)
+ foreach(_dst IN LISTS OLLAMA_MLX_JIT_INCLUDE_DESTINATIONS)
+ install(DIRECTORY "${_cuda_inc}/${_runtime_dir}"
+ DESTINATION ${_dst}
+ COMPONENT MLX)
+ endforeach()
endif()
endforeach()
- if(NOT WIN32 AND NOT APPLE)
- install(CODE "
- set(_link \"${CMAKE_INSTALL_PREFIX}/${OLLAMA_LIB_DIR}/include\")
- set(_target \"${OLLAMA_RUNNER_DIR}/include\")
- if(NOT EXISTS \${_link})
- execute_process(COMMAND \${CMAKE_COMMAND} -E create_symlink \${_target} \${_link})
- endif()
- " COMPONENT MLX)
- endif()
endif()
endif()
@@ -403,46 +455,85 @@ if(WIN32 AND TARGET dl)
COMPONENT MLX)
endif()
-# dlopen'd runtime libs, derived from MLX's dynamic.c/delayload.cpp
if(CUDAToolkit_FOUND)
- file(GLOB MLX_CUDA_LIBS
- "${CUDAToolkit_LIBRARY_DIR}/libcudart.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libcublas.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libnvrtc.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libnvrtc-builtins.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libcufft.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libcusolver.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libcusparse.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libnvJitLink.so*"
- "${CUDAToolkit_LIBRARY_DIR}/libcudnn*.so*")
+ set(MLX_SHARED_CUDA_LIBS)
+ set(MLX_PRIVATE_CUDA_LIBS)
+ set(MLX_CUDNN_LIBS)
if(WIN32)
- file(GLOB MLX_CUDA_DLLS
+ file(GLOB MLX_SHARED_CUDA_LIBS
+ "${CUDAToolkit_BIN_DIR}/cublas64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/cublasLt64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/cudart64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/nvfatbin_*.dll"
+ "${CUDAToolkit_BIN_DIR}/nvJitLink_*.dll"
+ "${CUDAToolkit_BIN_DIR}/nvrtc64_*.dll"
"${CUDAToolkit_BIN_DIR}/nvrtc-builtins64_*.dll"
- "${CUDAToolkit_BIN_DIR}/x64/nvrtc-builtins64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/cublas64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/cublasLt64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/cudart64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/nvfatbin_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/nvJitLink_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/nvrtc64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/nvrtc-builtins64_*.dll")
+ file(GLOB MLX_PRIVATE_CUDA_LIBS
+ "${CUDAToolkit_BIN_DIR}/cufft64_*.dll"
"${CUDAToolkit_BIN_DIR}/cusolver64_*.dll"
- "${CUDAToolkit_BIN_DIR}/x64/cusolver64_*.dll"
"${CUDAToolkit_BIN_DIR}/cusparse64_*.dll"
- "${CUDAToolkit_BIN_DIR}/x64/cusparse64_*.dll"
- "${CUDAToolkit_BIN_DIR}/nvJitLink_*.dll"
- "${CUDAToolkit_BIN_DIR}/x64/nvJitLink_*.dll")
- list(APPEND MLX_CUDA_LIBS ${MLX_CUDA_DLLS})
+ "${CUDAToolkit_BIN_DIR}/x64/cufft64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/cusolver64_*.dll"
+ "${CUDAToolkit_BIN_DIR}/x64/cusparse64_*.dll")
+ elseif(UNIX AND NOT APPLE)
+ file(GLOB MLX_SHARED_CUDA_LIBS
+ "${CUDAToolkit_LIBRARY_DIR}/libcudart.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libcublas.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libnvrtc.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libnvrtc-builtins.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libnvfatbin.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libnvJitLink.so*")
+ file(GLOB MLX_PRIVATE_CUDA_LIBS
+ "${CUDAToolkit_LIBRARY_DIR}/libcufft.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libcusolver.so*"
+ "${CUDAToolkit_LIBRARY_DIR}/libcusparse.so*")
+ endif()
+ if(MLX_SHARED_CUDA_LIBS)
+ list(REMOVE_DUPLICATES MLX_SHARED_CUDA_LIBS)
+ install(FILES ${MLX_SHARED_CUDA_LIBS}
+ DESTINATION ${OLLAMA_MLX_SHARED_CUDA_INSTALL_DIR}
+ COMPONENT MLX_SHARED)
endif()
- find_library(MLX_CUDNN_LIBRARY NAMES cudnn HINTS "$ENV{CUDNN_LIBRARY_PATH}")
- if(MLX_CUDNN_LIBRARY)
- get_filename_component(MLX_CUDNN_LIBRARY_DIR "${MLX_CUDNN_LIBRARY}" DIRECTORY)
- file(GLOB MLX_CUDNN_LIBS "${MLX_CUDNN_LIBRARY_DIR}/libcudnn*.so*")
- list(APPEND MLX_CUDA_LIBS ${MLX_CUDNN_LIBS})
+ if(MLX_PRIVATE_CUDA_LIBS)
+ list(REMOVE_DUPLICATES MLX_PRIVATE_CUDA_LIBS)
+ install(FILES ${MLX_PRIVATE_CUDA_LIBS}
+ DESTINATION ${OLLAMA_MLX_PRIVATE_CUDA_INSTALL_DIR}
+ COMPONENT MLX_VENDOR)
+ endif()
+ if(UNIX AND NOT APPLE)
+ find_library(MLX_CUDNN_LIBRARY NAMES cudnn
+ HINTS "${CUDNN_LIBRARY_PATH}" "$ENV{CUDNN_LIBRARY_PATH}")
+ if(MLX_CUDNN_LIBRARY)
+ get_filename_component(MLX_CUDNN_LIBRARY_DIR "${MLX_CUDNN_LIBRARY}" DIRECTORY)
+ file(GLOB MLX_CUDNN_LIBS "${MLX_CUDNN_LIBRARY_DIR}/libcudnn*.so*")
+ endif()
endif()
if(WIN32 AND _cudnn_root)
file(GLOB MLX_CUDNN_DLLS
"${_cudnn_root}/bin/${CUDAToolkit_VERSION_MAJOR}.0/cudnn*.dll"
"${_cudnn_root}/bin/x64/cudnn*.dll")
- list(APPEND MLX_CUDA_LIBS ${MLX_CUDNN_DLLS})
+ list(APPEND MLX_CUDNN_LIBS ${MLX_CUDNN_DLLS})
endif()
- if(MLX_CUDA_LIBS)
- install(FILES ${MLX_CUDA_LIBS}
- DESTINATION ${OLLAMA_INSTALL_DIR}
+ if(MLX_CUDNN_LIBS)
+ list(REMOVE_DUPLICATES MLX_CUDNN_LIBS)
+ install(FILES ${MLX_CUDNN_LIBS}
+ DESTINATION ${OLLAMA_MLX_CUDNN_INSTALL_DIR}
COMPONENT MLX_VENDOR)
endif()
endif()
+
+if(OLLAMA_MLX_OVERLAY_PACKAGE)
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/README_MLX.txt"
+ "Extract this MLX package to the same location as the Ollama base package.\n")
+ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/README_MLX.txt"
+ DESTINATION .
+ COMPONENT MLX)
+endif()
diff --git a/mlx/dynamic.go b/mlx/dynamic.go
index 990c0bf3f..101162b04 100644
--- a/mlx/dynamic.go
+++ b/mlx/dynamic.go
@@ -7,10 +7,10 @@ import "C"
import (
"fmt"
- "io/fs"
"log/slog"
"os"
"path/filepath"
+ "regexp"
"runtime"
"sort"
"strconv"
@@ -46,16 +46,27 @@ func LoadedLibraryPath() (string, error) {
return "", fmt.Errorf("MLX dynamic library not loaded")
}
-// tryLoadFromDir searches a directory for the mlxc shared library and loads it.
-func tryLoadFromDir(dir string) bool {
+// mlxLibraryCandidates returns the mlxc shared library in dir, if present.
+func mlxLibraryCandidates(dir string) []string {
// On Windows, MSVC produces mlxc.dll (no lib prefix)
// On Unix, it's libmlxc.so or libmlxc.dylib
- pattern := "libmlxc.*"
- if runtime.GOOS == "windows" {
- pattern = "mlxc.*"
+ name := "libmlxc.so"
+ switch runtime.GOOS {
+ case "windows":
+ name = "mlxc.dll"
+ case "darwin":
+ name = "libmlxc.dylib"
+ }
+ if info, err := os.Stat(filepath.Join(dir, name)); err != nil || !info.Mode().IsRegular() {
+ return nil
}
- matches, err := fs.Glob(os.DirFS(dir), pattern)
- if err != nil || len(matches) == 0 {
+ return []string{name}
+}
+
+// tryLoadFromDir searches a directory for the mlxc shared library and loads it.
+func tryLoadFromDir(dir string) bool {
+ matches := mlxLibraryCandidates(dir)
+ if len(matches) == 0 {
return false
}
for _, match := range matches {
@@ -167,24 +178,100 @@ func repoBuildDirs() []string {
return dirs
}
-// prependLibraryPath prepends dir to the platform's dynamic library search
-// path so the linker finds colocated libmlx before any stale copies.
-// Called once after successful library load.
-func prependLibraryPath(dir string) {
- var envVar string
+func libraryPathEnvVar() string {
switch runtime.GOOS {
+ case "windows":
+ return "PATH"
case "darwin":
- envVar = "DYLD_LIBRARY_PATH"
+ return "DYLD_LIBRARY_PATH"
case "linux":
- envVar = "LD_LIBRARY_PATH"
+ return "LD_LIBRARY_PATH"
default:
- return
+ return ""
+ }
+}
+
+// prependLibraryPaths prepends dirs to the platform's dynamic library search
+// path so MLX and any shared sibling backend dependencies resolve before stale
+// system copies. It returns a restore function for failed speculative loads.
+func prependLibraryPaths(dirs ...string) func() {
+ envVar := libraryPathEnvVar()
+ if envVar == "" {
+ return func() {}
+ }
+ oldValue, hadValue := os.LookupEnv(envVar)
+
+ var paths []string
+ seen := map[string]bool{}
+ add := func(path string) {
+ if path == "" {
+ return
+ }
+ path = filepath.Clean(path)
+ if !seen[path] {
+ seen[path] = true
+ paths = append(paths, path)
+ }
+ }
+ for _, dir := range dirs {
+ add(dir)
}
if existing := os.Getenv(envVar); existing != "" {
- os.Setenv(envVar, dir+string(filepath.ListSeparator)+existing)
+ for _, dir := range filepath.SplitList(existing) {
+ add(dir)
+ }
+ }
+ if len(paths) == 0 {
+ return func() {}
+ }
+ os.Setenv(envVar, strings.Join(paths, string(filepath.ListSeparator)))
+ return func() {
+ if hadValue {
+ os.Setenv(envVar, oldValue)
+ } else {
+ os.Unsetenv(envVar)
+ }
+ }
+}
+
+func mlxLibraryPathsForDir(dir string) []string {
+ paths := []string{dir}
+ if depDir, ok := dependencyDir(dir); ok {
+ paths = append(paths, depDir)
+ }
+ return paths
+}
+
+func tryLoadFromMLXDir(dir string) bool {
+ restore := prependLibraryPaths(mlxLibraryPathsForDir(dir)...)
+ if tryLoadFromDir(dir) {
+ return true
+ }
+ restore()
+ return false
+}
+
+func prependLoadedLibraryPath() {
+ if initLoadedPath == "" {
+ return
+ }
+ dir := filepath.Dir(initLoadedPath)
+ if strings.HasPrefix(filepath.Base(dir), "mlx_") {
+ prependLibraryPaths(mlxLibraryPathsForDir(dir)...)
+ setBundledCUDAToolkitPath(dir)
} else {
- os.Setenv(envVar, dir)
+ prependLibraryPaths(dir)
+ }
+}
+
+func setBundledCUDAToolkitPath(dir string) {
+ cudaPath, ok := cudaToolkitDirForMLXDir(dir)
+ if !ok {
+ return
}
+ os.Setenv("CUDA_PATH", cudaPath)
+ os.Setenv("CUDA_HOME", cudaPath)
+ slog.Debug("MLX CUDA headers", "CUDA_PATH", cudaPath)
}
func init() {
@@ -208,13 +295,13 @@ func init() {
return
}
- prependLibraryPath(filepath.Dir(initLoadedPath))
+ prependLoadedLibraryPath()
}
func findMLXLibrary(forcedVariant string) bool {
for _, root := range libOllamaRoots() {
if forcedVariant != "" {
- if tryLoadFromDir(filepath.Join(root, forcedVariant)) {
+ if tryLoadFromMLXDir(filepath.Join(root, forcedVariant)) {
return true
}
} else {
@@ -229,6 +316,10 @@ func findMLXLibrary(forcedVariant string) bool {
return false
}
+// mlxVariantPattern matches the variant directory names we ship, e.g.
+// mlx_metal_v4, mlx_cuda_v13.
+var mlxVariantPattern = regexp.MustCompile(`^mlx_(metal|cuda)_v\d+$`)
+
// tryLoadFromMLXSubdirs globs for mlx_* subdirs within dir, filters out
// incompatible variants, tries the remainder in reverse sorted order (so
// higher-versioned variants are preferred), and returns true on first
@@ -241,11 +332,15 @@ func tryLoadFromMLXSubdirs(dir string) bool {
// Reverse sort: mlx_metal_v4 before mlx_metal_v3, mlx_cuda_v13 before v12
sort.Sort(sort.Reverse(sort.StringSlice(mlxDirs)))
for _, mlxDir := range mlxDirs {
+ if !mlxVariantPattern.MatchString(filepath.Base(mlxDir)) {
+ slog.Debug("skipping unrecognized MLX variant", "dir", mlxDir)
+ continue
+ }
if !isCompatibleMLXVariant(filepath.Base(mlxDir)) {
slog.Debug("skipping incompatible MLX variant", "dir", mlxDir)
continue
}
- if tryLoadFromDir(mlxDir) {
+ if tryLoadFromMLXDir(mlxDir) {
return true
}
}
diff --git a/mlx/path.go b/mlx/path.go
new file mode 100644
index 000000000..403aa2f12
--- /dev/null
+++ b/mlx/path.go
@@ -0,0 +1,81 @@
+package mlx
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// LibraryPathsForLoadedLibrary returns the runtime library search paths needed
+// for the selected MLX dynamic library.
+func LibraryPathsForLoadedLibrary(libraryPath string) []string {
+ return libraryPathsForLibrary(libraryPath)
+}
+
+// CUDAToolkitDirForLoadedLibrary returns the selected MLX CUDA variant
+// directory when it carries bundled CUDA headers for MLX JIT compilation.
+func CUDAToolkitDirForLoadedLibrary(libraryPath string) (string, bool) {
+ return cudaToolkitDirForLibrary(libraryPath)
+}
+
+func libraryPathsForLibrary(libraryPath string) []string {
+ var paths []string
+ seen := map[string]bool{}
+ add := func(path string) {
+ if path == "" {
+ return
+ }
+ path = filepath.Clean(path)
+ if !seen[path] {
+ seen[path] = true
+ paths = append(paths, path)
+ }
+ }
+
+ dir := filepath.Dir(libraryPath)
+ add(dir)
+ if strings.HasPrefix(filepath.Base(dir), "mlx_") {
+ if depDir, ok := dependencyDir(dir); ok {
+ add(depDir)
+ }
+ }
+
+ return paths
+}
+
+// dependencyDir maps an MLX CUDA variant directory to its shared dependency
+// directory. Variants like mlx_cuda_v12 automatically use cuda_v12 when
+// present; non-CUDA variants do not have a sibling dependency payload.
+func dependencyDir(mlxDir string) (string, bool) {
+ name := filepath.Base(mlxDir)
+ cudaName, ok := strings.CutPrefix(name, "mlx_cuda_")
+ if !ok || cudaName == "" {
+ return "", false
+ }
+ depName := "cuda_" + cudaName
+
+ depDir := filepath.Join(filepath.Dir(mlxDir), depName)
+ if !dirExists(depDir) {
+ return "", false
+ }
+ return depDir, true
+}
+
+func cudaToolkitDirForLibrary(libraryPath string) (string, bool) {
+ return cudaToolkitDirForMLXDir(filepath.Dir(libraryPath))
+}
+
+func cudaToolkitDirForMLXDir(mlxDir string) (string, bool) {
+ if !strings.HasPrefix(filepath.Base(mlxDir), "mlx_cuda_") {
+ return "", false
+ }
+ if !dirExists(filepath.Join(mlxDir, "include")) {
+ return "", false
+ }
+ return filepath.Clean(mlxDir), true
+}
+
+func dirExists(path string) bool {
+ info, err := os.Stat(path)
+ return err == nil && info.IsDir()
+}
diff --git a/mlx/path_test.go b/mlx/path_test.go
new file mode 100644
index 000000000..61c6c9e8b
--- /dev/null
+++ b/mlx/path_test.go
@@ -0,0 +1,128 @@
+package mlx
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+)
+
+func TestLibraryPathsForLoadedLibraryAddsOnlySelectedMLXDependency(t *testing.T) {
+ root := t.TempDir()
+ for _, dir := range []string{
+ "mlx_cuda_v13",
+ "cuda_v13",
+ "mlx_cuda_v12",
+ "cuda_v12",
+ } {
+ if err := os.Mkdir(filepath.Join(root, dir), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ got := LibraryPathsForLoadedLibrary(filepath.Join(root, "mlx_cuda_v13", "mlxc.dll"))
+ want := []string{
+ filepath.Join(root, "mlx_cuda_v13"),
+ filepath.Join(root, "cuda_v13"),
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("LibraryPathsForLoadedLibrary() = %#v, want %#v", got, want)
+ }
+}
+
+func TestLibraryPathsForLoadedLibraryAddsStandaloneDir(t *testing.T) {
+ root := t.TempDir()
+ libDir := filepath.Join(root, "standalone")
+ got := LibraryPathsForLoadedLibrary(filepath.Join(libDir, "libmlxc.so"))
+ want := []string{libDir}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("LibraryPathsForLoadedLibrary() = %#v, want %#v", got, want)
+ }
+}
+
+func TestDependencyDirRequiresExistingSibling(t *testing.T) {
+ root := t.TempDir()
+ mlxDir := filepath.Join(root, "mlx_cuda_v13")
+ if err := os.Mkdir(mlxDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+
+ if depDir, ok := dependencyDir(mlxDir); ok {
+ t.Fatalf("dependencyDir() = %q, true; want missing dependency", depDir)
+ }
+
+ cudaDir := filepath.Join(root, "cuda_v13")
+ if err := os.Mkdir(cudaDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if depDir, ok := dependencyDir(mlxDir); !ok || depDir != cudaDir {
+ t.Fatalf("dependencyDir() = %q, %v; want %q, true", depDir, ok, cudaDir)
+ }
+}
+
+func TestDependencyDirRejectsNonCUDAVariant(t *testing.T) {
+ root := t.TempDir()
+ for _, dir := range []string{"mlx_metal_v3", "metal_v3"} {
+ if err := os.Mkdir(filepath.Join(root, dir), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if depDir, ok := dependencyDir(filepath.Join(root, "mlx_metal_v3")); ok {
+ t.Fatalf("dependencyDir() = %q, true; want non-CUDA variant rejected", depDir)
+ }
+}
+
+func TestCUDAToolkitDirForLoadedLibraryRequiresMLXCUDAInclude(t *testing.T) {
+ root := t.TempDir()
+ for _, dir := range []string{
+ "mlx_cuda_v13",
+ "mlx_test_v1",
+ } {
+ if err := os.Mkdir(filepath.Join(root, dir), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ libraryPath := filepath.Join(root, "mlx_cuda_v13", "mlxc.dll")
+
+ if cudaDir, ok := CUDAToolkitDirForLoadedLibrary(libraryPath); ok {
+ t.Fatalf("CUDAToolkitDirForLoadedLibrary() = %q, true; want no CUDA headers", cudaDir)
+ }
+
+ want := filepath.Join(root, "mlx_cuda_v13")
+ if err := os.Mkdir(filepath.Join(want, "include"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if cudaDir, ok := CUDAToolkitDirForLoadedLibrary(libraryPath); !ok || cudaDir != want {
+ t.Fatalf("CUDAToolkitDirForLoadedLibrary() = %q, %v; want %q, true", cudaDir, ok, want)
+ }
+}
+
+func TestCUDAToolkitDirForLoadedLibraryUsesSelectedCUDAName(t *testing.T) {
+ root := t.TempDir()
+ for _, dir := range []string{
+ filepath.Join(root, "mlx_cuda_v12", "include"),
+ filepath.Join(root, "mlx_cuda_v13", "include"),
+ } {
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ want := filepath.Join(root, "mlx_cuda_v12")
+ if cudaDir, ok := CUDAToolkitDirForLoadedLibrary(filepath.Join(want, "mlxc.dll")); !ok || cudaDir != want {
+ t.Fatalf("CUDAToolkitDirForLoadedLibrary() = %q, %v; want %q, true", cudaDir, ok, want)
+ }
+}
+
+func TestCUDAToolkitDirForMLXDirRejectsNonCUDAVariants(t *testing.T) {
+ root := t.TempDir()
+ otherDir := filepath.Join(root, "mlx_test_v1")
+ if err := os.MkdirAll(filepath.Join(otherDir, "include"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+
+ if cudaDir, ok := cudaToolkitDirForMLXDir(otherDir); ok {
+ t.Fatalf("cudaToolkitDirForMLXDir() = %q, true; want rejected non-CUDA variant", cudaDir)
+ }
+}
diff --git a/mlxrunner/client.go b/mlxrunner/client.go
index 6c92727b8..649473622 100644
--- a/mlxrunner/client.go
+++ b/mlxrunner/client.go
@@ -438,6 +438,10 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D
// Spawn subprocess: ollama runner --model <name> --port <port>
cmd := exec.Command(exe, "runner", "--model", c.modelName, "--port", strconv.Itoa(port))
cmd.Env = os.Environ()
+ loadedLibraryPath, err := mlx.LoadedLibraryPath()
+ if err != nil {
+ return nil, fmt.Errorf("unable to determine loaded MLX library path: %w", err)
+ }
// Keep Metal weights resident between requests until MLX fixes idle eviction.
if runtime.GOOS == "darwin" {
@@ -457,16 +461,8 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D
}
if libPathEnvVar != "" {
- libraryPaths := []string{ml.LibOllamaPath}
- if mlxDirs, err := filepath.Glob(filepath.Join(ml.LibOllamaPath, "mlx_*")); err == nil {
- libraryPaths = append(libraryPaths, mlxDirs...)
- }
-
- if existingPath, ok := os.LookupEnv(libPathEnvVar); ok {
- libraryPaths = append(libraryPaths, filepath.SplitList(existingPath)...)
- }
-
- pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))
+ libraryPaths := mlx.LibraryPathsForLoadedLibrary(loadedLibraryPath)
+ pathEnvVal := libraryPathEnv(libraryPaths, os.Getenv(libPathEnvVar))
found := false
for i := range cmd.Env {
@@ -490,15 +486,10 @@ func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.D
// MLX resolves headers via $CUDA_PATH/include/*.h (and checks CUDA_HOME first).
// Always use bundled headers to avoid version mismatches with any
// system-installed CUDA toolkit.
- if mlxDirs, err := filepath.Glob(filepath.Join(ml.LibOllamaPath, "mlx_cuda_*")); err == nil {
- for _, d := range mlxDirs {
- if _, err := os.Stat(filepath.Join(d, "include")); err == nil {
- setEnv(cmd, "CUDA_PATH", d)
- setEnv(cmd, "CUDA_HOME", d)
- slog.Debug("mlx subprocess CUDA headers", "CUDA_PATH", d)
- break
- }
- }
+ if cudaPath, ok := mlx.CUDAToolkitDirForLoadedLibrary(loadedLibraryPath); ok {
+ setEnv(cmd, "CUDA_PATH", cudaPath)
+ setEnv(cmd, "CUDA_HOME", cudaPath)
+ slog.Debug("mlx subprocess CUDA headers", "CUDA_PATH", cudaPath)
}
status := llm.NewStatusWriter(os.Stderr)
@@ -653,3 +644,31 @@ func setEnv(cmd *exec.Cmd, key, value string) {
}
cmd.Env = append(cmd.Env, entry)
}
+
+func libraryPathEnv(preferred []string, existing string) string {
+ var paths []string
+ seen := map[string]bool{}
+ add := func(path string) {
+ if path == "" {
+ return
+ }
+ path = filepath.Clean(path)
+ key := path
+ if runtime.GOOS == "windows" {
+ key = strings.ToUpper(key)
+ }
+ if !seen[key] {
+ seen[key] = true
+ paths = append(paths, path)
+ }
+ }
+
+ for _, path := range preferred {
+ add(path)
+ }
+ for _, path := range filepath.SplitList(existing) {
+ add(path)
+ }
+
+ return strings.Join(paths, string(filepath.ListSeparator))
+}
diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh
index b4485e75a..ac85b069d 100755
--- a/scripts/build_linux.sh
+++ b/scripts/build_linux.sh
@@ -40,14 +40,6 @@ docker buildx build \
-f Dockerfile \
.
-# Run deduplication for each platform output directory
-if echo $PLATFORM | grep "," > /dev/null ; then
- $(dirname $0)/deduplicate_cuda_libs.sh "./dist/linux_amd64"
- $(dirname $0)/deduplicate_cuda_libs.sh "./dist/linux_arm64"
-elif echo $PLATFORM | grep "amd64\|arm64" > /dev/null ; then
- $(dirname $0)/deduplicate_cuda_libs.sh "./dist"
-fi
-
# buildx behavior changes for single vs. multiplatform
echo "Compressing linux tar bundles..."
if echo $PLATFORM | grep "," > /dev/null ; then
diff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1
index 0b3c3b3ba..5dbd23bf8 100644
--- a/scripts/build_windows.ps1
+++ b/scripts/build_windows.ps1
@@ -840,7 +840,8 @@ function mlxCuda13 {
$cudaFlags += "-DCMAKE_CUDA_FLAGS=$env:OLLAMA_CMAKE_CUDA_FLAGS"
}
$cudaToolsetArgs = cudaCMakeArgs $cuda
- $configureArgs = @("-S", ".", "-B", "build\mlx_cuda_v$cudaMajorVer", "-DOLLAMA_MLX_BACKENDS=cuda_v$cudaMajorVer") + $cudaToolsetArgs + $cudaFlags + @("-DOLLAMA_PAYLOAD_INSTALL_PREFIX=$script:DIST_DIR", "--install-prefix", "$script:DIST_DIR")
+ $mlxDistDir = "${script:SRC_DIR}\dist\windows-amd64-mlx"
+ $configureArgs = @("-S", ".", "-B", "build\mlx_cuda_v$cudaMajorVer", "-DCMAKE_BUILD_TYPE=Release", "-DOLLAMA_MLX_BACKENDS=cuda_v$cudaMajorVer") + $cudaToolsetArgs + $cudaFlags + @("-DOLLAMA_PAYLOAD_INSTALL_PREFIX=$script:DIST_DIR", "-DOLLAMA_MLX_PAYLOAD_INSTALL_PREFIX=$mlxDistDir", "--install-prefix", "$script:DIST_DIR")
& cmake @configureArgs
if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
$buildArgs = @("--build", "build\mlx_cuda_v$cudaMajorVer", "--target", "ollama-mlx-cuda_v$cudaMajorVer", "--config", "Release", "--parallel", "$script:JOBS")
@@ -1245,8 +1246,8 @@ function zip {
$jobs += newDependencyAuditJob "${distDir}\windows-amd64-rocm" "windows-amd64-rocm" "${distDir}\dependency-audit-windows-amd64-rocm.txt" $amd64Dir
}
- # Stage MLX into its own directory for independent compression
- if (stageComponents $amd64Dir "${distDir}\windows-amd64-mlx" "mlx_*" "MLX") {
+ # CMake installs the MLX overlay into its own staging directory.
+ if (Test-Path -Path "${distDir}\windows-amd64-mlx\lib\ollama") {
Write-Output "Generating ${distDir}\ollama-windows-amd64-mlx.zip"
$jobs += newZipJob "${distDir}\windows-amd64-mlx" "${distDir}\ollama-windows-amd64-mlx.zip"
$jobs += newDependencyAuditJob "${distDir}\windows-amd64-mlx" "windows-amd64-mlx" "${distDir}\dependency-audit-windows-amd64-mlx.txt" $amd64Dir
@@ -1287,7 +1288,6 @@ function zip {
} finally {
# Always restore staged components back into the main tree
restoreComponents $amd64Dir "${distDir}\windows-amd64-rocm"
- restoreComponents $amd64Dir "${distDir}\windows-amd64-mlx"
}
}
diff --git a/scripts/deduplicate_cuda_libs.sh b/scripts/deduplicate_cuda_libs.sh
deleted file mode 100755
index f8d13cd3d..000000000
--- a/scripts/deduplicate_cuda_libs.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/sh
-#
-# Deduplicate CUDA libraries across mlx_* and cuda_* directories
-# This script finds identical .so* files in mlx_cuda_* directories that exist
-# in corresponding cuda_* directories and replaces them with symlinks.
-#
-
-set -eu
-
-if [ $# -eq 0 ]; then
- echo "ERROR: No directory specified" >&2
- echo "Usage: $0 <base_directory>" >&2
- exit 1
-fi
-
-base_dir="$1"
-
-if [ ! -d "${base_dir}" ]; then
- echo "ERROR: Directory ${base_dir} does not exist" >&2
- exit 1
-fi
-
-echo "Deduplicating CUDA libraries in ${base_dir}..."
-
-# Find all mlx_cuda_* directories
-for mlx_dir in "${base_dir}"/lib/ollama/mlx_cuda_*; do
- [ -d "${mlx_dir}" ] || continue
-
- # Extract CUDA version (e.g., v12, v13)
- cuda_version=$(basename "${mlx_dir}" | sed 's/mlx_cuda_//')
- cuda_dir="${base_dir}/lib/ollama/cuda_${cuda_version}"
-
- # Skip if corresponding cuda_* directory doesn't exist
- [ -d "${cuda_dir}" ] || continue
-
- echo " Checking ${mlx_dir} against ${cuda_dir}..."
-
- # Find all .so* files in mlx directory
- find "${mlx_dir}" -type f -name "*.so*" | while read mlx_file; do
- filename=$(basename "${mlx_file}")
- cuda_file="${cuda_dir}/${filename}"
-
- # Skip if file doesn't exist in cuda directory
- [ -f "${cuda_file}" ] || continue
-
- # Compare checksums
- mlx_sum=$(sha256sum "${mlx_file}" | awk '{print $1}')
- cuda_sum=$(sha256sum "${cuda_file}" | awk '{print $1}')
-
- if [ "${mlx_sum}" = "${cuda_sum}" ]; then
- echo " Deduplicating ${filename}"
- # Calculate relative path from mlx_dir to cuda_dir
- rel_path="../cuda_${cuda_version}/${filename}"
- rm -f "${mlx_file}"
- ln -s "${rel_path}" "${mlx_file}"
- fi
- done
-done
-
-echo "Deduplication complete"