Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Add v12 + v13 cuda support (#12000)
Daniel Hiltgen17a023f34b9e
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yamlindex 40871e644..902fa9ccc 100644--- a/.github/workflows/release.yaml+++ b/.github/workflows/release.yaml@@ -67,12 +67,21 @@ jobs:install: https://developer.download.nvidia.com/compute/cuda/12.8.0/local_installers/cuda_12.8.0_571.96_windows.execuda-version: '12.8'flags: ''+ runner_dir: 'cuda_v12'+ - os: windows+ arch: amd64+ preset: 'CUDA 13'+ install: https://developer.download.nvidia.com/compute/cuda/13.0.0/local_installers/cuda_13.0.0_windows.exe+ cuda-version: '13.0'+ flags: ''+ runner_dir: 'cuda_v13'- os: windowsarch: amd64preset: 'ROCm 6'install: https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q4-WinSvr2022-For-HIP.exerocm-version: '6.2'flags: '-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_FLAGS="-parallel-jobs=4 -Wno-ignored-attributes -Wno-deprecated-pragma" -DCMAKE_CXX_FLAGS="-parallel-jobs=4 -Wno-ignored-attributes -Wno-deprecated-pragma"'+ runner_dir: ''runs-on: ${{ matrix.arch == 'arm64' && format('{0}-{1}', matrix.os, matrix.arch) || matrix.os }}environment: releaseenv:@@ -138,7 +147,7 @@ jobs:run: |Import-Module 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'Enter-VsDevShell -VsInstallPath 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise' -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -no_logo'- cmake --preset "${{ matrix.preset }}" ${{ matrix.flags }}+ cmake --preset "${{ matrix.preset }}" ${{ matrix.flags }} -DOLLAMA_RUNNER_DIR="${{ matrix.runner_dir }}"cmake --build --parallel --preset "${{ matrix.preset }}"cmake --install build --component "${{ startsWith(matrix.preset, 'CUDA ') && 'CUDA' || startsWith(matrix.preset, 'ROCm ') && 'HIP' || 'CPU' }}" --strip --parallel 8env:@@ -232,7 +241,7 @@ jobs:case "$COMPONENT" inbin/ollama) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;lib/ollama/*.so*) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;- lib/ollama/cuda_sbsa) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;+ lib/ollama/cuda_v*) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;lib/ollama/cuda_jetpack5) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-jetpack5.tar.in ;;lib/ollama/cuda_jetpack6) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-jetpack6.tar.in ;;lib/ollama/rocm) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-rocm.tar.in ;;diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yamlindex 4d8cf773c..a10ad37a9 100644--- a/.github/workflows/test.yaml+++ b/.github/workflows/test.yaml@@ -46,7 +46,7 @@ jobs:include:- preset: CPU- preset: CUDA- container: nvidia/cuda:12.8.1-devel-ubuntu22.04+ container: nvidia/cuda:13.0.0-devel-ubuntu22.04flags: '-DCMAKE_CUDA_ARCHITECTURES=87'- preset: ROCmcontainer: rocm/dev-ubuntu-22.04:6.1.2@@ -78,7 +78,7 @@ jobs:include:- preset: CPU- preset: CUDA- install: https://developer.download.nvidia.com/compute/cuda/12.8.0/local_installers/cuda_12.8.0_571.96_windows.exe+ install: https://developer.download.nvidia.com/compute/cuda/13.0.0/local_installers/cuda_13.0.0_windows.exeflags: '-DCMAKE_CUDA_ARCHITECTURES=80'- preset: ROCminstall: https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q4-WinSvr2022-For-HIP.exe@@ -102,7 +102,7 @@ jobs:$ErrorActionPreference = "Stop"if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"- Start-Process -FilePath .\install.exe -ArgumentList (@("-s", "cudart_12.8", "nvcc_12.8", "cublas_12.8", "cublas_dev_12.8")) -NoNewWindow -Wait+ Start-Process -FilePath .\install.exe -ArgumentList (@("-s", "cudart_13.0", "nvcc_13.0", "cublas_13.0", "cublas_dev_13.0")) -NoNewWindow -Wait}$cudaPath = (Resolve-Path "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*").pathdiff --git a/CMakeLists.txt b/CMakeLists.txtindex d62c8f99f..7cce5e4b1 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -25,7 +25,7 @@ set(GGML_LLAMAFILE ON)set(GGML_CUDA_PEER_MAX_BATCH_SIZE 128)set(GGML_CUDA_GRAPHS ON)set(GGML_CUDA_FA ON)-set(GGML_CUDA_COMPRESSION_MODE default)+set(GGML_CUDA_COMPRESSION_MODE size)if((CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_OSX_ARCHITECTURES MATCHES "arm64")OR (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm|aarch64|ARM64|ARMv[0-9]+"))@@ -38,7 +38,7 @@ if (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64")endif()set(OLLAMA_BUILD_DIR ${CMAKE_BINARY_DIR}/lib/ollama)-set(OLLAMA_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/ollama)+set(OLLAMA_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/ollama/${OLLAMA_RUNNER_DIR})set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OLLAMA_BUILD_DIR})set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${OLLAMA_BUILD_DIR})@@ -81,7 +81,7 @@ if(CMAKE_CUDA_COMPILER)add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src/ggml-cuda)install(TARGETS ggml-cudaRUNTIME_DEPENDENCIES- DIRECTORIES ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_LIBRARY_DIR}+ DIRECTORIES ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_BIN_DIR}/x64 ${CUDAToolkit_LIBRARY_DIR}PRE_INCLUDE_REGEXES cublas cublasLt cudartPRE_EXCLUDE_REGEXES ".*"RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT CUDAdiff --git a/CMakePresets.json b/CMakePresets.jsonindex ab2cfe9d6..51190c719 100644--- a/CMakePresets.json+++ b/CMakePresets.json@@ -18,6 +18,14 @@"name": "CUDA","inherits": [ "Default" ]},+ {+ "name": "CUDA 11",+ "inherits": [ "CUDA" ],+ "cacheVariables": {+ "CMAKE_CUDA_ARCHITECTURES": "50-virtual;60-virtual;61-virtual;70-virtual;75-virtual;80-virtual;86-virtual;87-virtual;89-virtual;90-virtual",+ "CMAKE_CUDA_FLAGS": "-Wno-deprecated-gpu-targets -t 2"+ }+ },{"name": "CUDA 12","inherits": [ "CUDA" ],@@ -26,6 +34,14 @@"CMAKE_CUDA_FLAGS": "-Wno-deprecated-gpu-targets -t 2"}},+ {+ "name": "CUDA 13",+ "inherits": [ "CUDA" ],+ "cacheVariables": {+ "CMAKE_CUDA_ARCHITECTURES": "75-virtual;80-virtual;86-virtual;87-virtual;89-virtual;90-virtual;90a-virtual;100-virtual;110-virtual;120-virtual;121-virtual",+ "CMAKE_CUDA_FLAGS": "-t 2"+ }+ },{"name": "JetPack 5","inherits": [ "CUDA" ],@@ -72,11 +88,21 @@"configurePreset": "CUDA","targets": [ "ggml-cuda" ]},+ {+ "name": "CUDA 11",+ "inherits": [ "CUDA" ],+ "configurePreset": "CUDA 11"+ },{"name": "CUDA 12","inherits": [ "CUDA" ],"configurePreset": "CUDA 12"},+ {+ "name": "CUDA 13",+ "inherits": [ "CUDA" ],+ "configurePreset": "CUDA 13"+ },{"name": "JetPack 5","inherits": [ "CUDA" ],diff --git a/Dockerfile b/Dockerfileindex 0dc3c1267..c84b52392 100644--- a/Dockerfile+++ b/Dockerfile@@ -39,15 +39,35 @@ RUN --mount=type=cache,target=/root/.ccache \&& cmake --build --parallel --preset 'CPU' \&& cmake --install build --component CPU --strip --parallel 8+FROM base AS cuda-11+ARG CUDA11VERSION=11.8+RUN dnf install -y cuda-toolkit-${CUDA11VERSION//./-}+ENV PATH=/usr/local/cuda-11/bin:$PATH+RUN --mount=type=cache,target=/root/.ccache \+ cmake --preset 'CUDA 11' -DOLLAMA_RUNNER_DIR="cuda_v11" \+ && cmake --build --parallel --preset 'CUDA 11' \+ && cmake --install build --component CUDA --strip --parallel 8+FROM base AS cuda-12ARG CUDA12VERSION=12.8RUN dnf install -y cuda-toolkit-${CUDA12VERSION//./-}ENV PATH=/usr/local/cuda-12/bin:$PATHRUN --mount=type=cache,target=/root/.ccache \- cmake --preset 'CUDA 12' \+ cmake --preset 'CUDA 12' -DOLLAMA_RUNNER_DIR="cuda_v12"\&& cmake --build --parallel --preset 'CUDA 12' \&& cmake --install build --component CUDA --strip --parallel 8++FROM base AS cuda-13+ARG CUDA13VERSION=13.0+RUN dnf install -y cuda-toolkit-${CUDA13VERSION//./-}+ENV PATH=/usr/local/cuda-13/bin:$PATH+RUN --mount=type=cache,target=/root/.ccache \+ cmake --preset 'CUDA 13' -DOLLAMA_RUNNER_DIR="cuda_v13" \+ && cmake --build --parallel --preset 'CUDA 13' \+ && cmake --install build --component CUDA --strip --parallel 8++FROM base AS rocm-6ENV PATH=/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin:/opt/rocm/hcc/bin:$PATHRUN --mount=type=cache,target=/root/.ccache \@@ -92,10 +112,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \go build -trimpath -buildmode=pie -o /bin/ollama .FROM --platform=linux/amd64 scratch AS amd64-COPY --from=cuda-12 dist/lib/ollama /lib/ollama+# COPY --from=cuda-11 dist/lib/ollama/ /lib/ollama/+COPY --from=cuda-12 dist/lib/ollama /lib/ollama/+COPY --from=cuda-13 dist/lib/ollama/ /lib/ollama/FROM --platform=linux/arm64 scratch AS arm64-COPY --from=cuda-12 dist/lib/ollama /lib/ollama/cuda_sbsa+# COPY --from=cuda-11 dist/lib/ollama/ /lib/ollama/+COPY --from=cuda-12 dist/lib/ollama /lib/ollama/+COPY --from=cuda-13 dist/lib/ollama/ /lib/ollama/COPY --from=jetpack-5 dist/lib/ollama /lib/ollama/cuda_jetpack5COPY --from=jetpack-6 dist/lib/ollama /lib/ollama/cuda_jetpack6diff --git a/discover/cuda_common.go b/discover/cuda_common.goindex b539f6b32..ca008af63 100644--- a/discover/cuda_common.go+++ b/discover/cuda_common.go@@ -43,14 +43,15 @@ func cudaVariant(gpuInfo CudaGPUInfo) string {}}}- return "sbsa"}- // driver 12.0 has problems with the cuda v12 library, so run v11 on those older drivers- if gpuInfo.DriverMajor < 12 || (gpuInfo.DriverMajor == 12 && gpuInfo.DriverMinor == 0) {- // The detected driver is older than Feb 2023- slog.Warn("old CUDA driver detected - please upgrade to a newer driver", "version", fmt.Sprintf("%d.%d", gpuInfo.DriverMajor, gpuInfo.DriverMinor))- return "v11"+ if gpuInfo.DriverMajor < 13 {+ // The detected driver is older than 580 (Aug 2025)+ // Warn if their CC is compatible with v13 and they should upgrade their driver to get better performance+ if gpuInfo.computeMajor > 7 || (gpuInfo.computeMajor == 7 && gpuInfo.computeMinor >= 5) {+ slog.Warn("old CUDA driver detected - please upgrade to a newer driver for best performance", "version", fmt.Sprintf("%d.%d", gpuInfo.DriverMajor, gpuInfo.DriverMinor))+ }+ return "v12"}- return "v12"+ return "v13"}diff --git a/docs/linux.md b/docs/linux.mdindex 9a156d1dc..ce5ed860b 100644--- a/docs/linux.md+++ b/docs/linux.md@@ -11,12 +11,13 @@ curl -fsSL https://ollama.com/install.sh | sh## Manual install> [!NOTE]-> If you are upgrading from a prior version, you should remove the old libraries with `sudo rm -rf /usr/lib/ollama` first.+> If you are upgrading from a prior version, you **MUST** remove the old libraries with `sudo rm -rf /usr/lib/ollama` first.Download and extract the package:```shellcurl -LO https://ollama.com/download/ollama-linux-amd64.tgz+sudo rm -rf /usr/lib/ollamasudo tar -C /usr -xzf ollama-linux-amd64.tgz```diff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1index 27f3eb9d4..37fe87961 100644--- a/scripts/build_windows.ps1+++ b/scripts/build_windows.ps1@@ -78,7 +78,7 @@ function checkEnv() {}-function buildOllama() {+function buildCPU() {mkdir -Force -path "${script:DIST_DIR}\"if ($script:ARCH -ne "arm64") {Remove-Item -ea 0 -recurse -force -path "${script:SRC_DIR}\dist\windows-${script:ARCH}"@@ -90,20 +90,72 @@ function buildOllama() {if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}& cmake --install build --component CPU --stripif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}+ }+}+function buildCUDA11() {+ # CUDA v11 claims to be compatible with MSVC 2022, but the latest updates are no longer compatible+ # 19.40 is the last compiler version that works, but recent udpates are 19.43+ # So this pins to MSVC 2019 for best compatibility+ mkdir -Force -path "${script:DIST_DIR}\"+ if ($script:ARCH -ne "arm64") {$hashEnv = @{}Get-ChildItem env: | foreach { $hashEnv[$_.Name] = $_.Value }- if ("$script:CUDA_DIRS".Contains("v12")) {- $hashEnv.Keys | foreach { if ($_.Contains("CUDA_PATH_V12")) { $v12="$_" }}- $env:CUDAToolkit_ROOT=$hashEnv[$v12]- write-host "Building CUDA v12 backend libraries"- & cmake --fresh --preset "CUDA 12" --install-prefix $script:DIST_DIR+ if ("$script:CUDA_DIRS".Contains("v11")) {+ $hashEnv.Keys | foreach { if ($_.Contains("CUDA_PATH_V11")) { $x=$hashEnv[$_]; if (test-path -literalpath "$x\bin\nvcc.exe" ) { $cuda=$x} }}+ write-host "Building CUDA v11 backend libraries $cuda"+ $env:CUDAToolkit_ROOT=$cuda+ & cmake --fresh --preset "CUDA 11" -T cuda="$cuda" -DCMAKE_CUDA_COMPILER="$cuda\bin\nvcc.exe" -G "Visual Studio 16 2019" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v11"+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}+ & cmake --build --preset "CUDA 11" --config Release --parallel $script:JOBS+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}+ & cmake --install build --component "CUDA" --strip+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}+ }+ }+}++function buildCUDA12() {+ mkdir -Force -path "${script:DIST_DIR}\"+ if ($script:ARCH -ne "arm64") {+ $hashEnv = @{}+ Get-ChildItem env: | foreach { $hashEnv[$_.Name] = $_.Value }+ if ("$script:CUDA_DIRS".Contains("v12.8")) {+ $hashEnv.Keys | foreach { if ($_.Contains("CUDA_PATH_V12_8")) { $x=$hashEnv[$_]; if (test-path -literalpath "$x\bin\nvcc.exe" ) { $cuda=$x} }}+ write-host "Building CUDA v12 backend libraries $cuda"+ $env:CUDAToolkit_ROOT=$cuda+ & cmake --fresh --preset "CUDA 12" -T cuda="$cuda" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v12"if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}& cmake --build --preset "CUDA 12" --config Release --parallel $script:JOBSif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}& cmake --install build --component "CUDA" --stripif ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}}+ }+}++function buildCUDA13() {+ mkdir -Force -path "${script:DIST_DIR}\"+ if ($script:ARCH -ne "arm64") {+ $hashEnv = @{}+ Get-ChildItem env: | foreach { $hashEnv[$_.Name] = $_.Value }+ if ("$script:CUDA_DIRS".Contains("v13")) {+ $hashEnv.Keys | foreach { if ($_.Contains("CUDA_PATH_V13")) { $x=$hashEnv[$_]; if (test-path -literalpath "$x\bin\nvcc.exe" ) { $cuda=$x} }}+ $env:CUDAToolkit_ROOT=$cuda+ write-host "Building CUDA v13 backend libraries $cuda"+ & cmake --fresh --preset "CUDA 13" -T cuda="$cuda" --install-prefix $script:DIST_DIR -DOLLAMA_RUNNER_DIR="cuda_v13"+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}+ & cmake --build --preset "CUDA 13" --config Release --parallel $script:JOBS+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}+ & cmake --install build --component "CUDA" --strip+ if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}+ }+ }+}++function buildROCm() {+ mkdir -Force -path "${script:DIST_DIR}\"+ if ($script:ARCH -ne "arm64") {if ($env:HIP_PATH) {write-host "Building ROCm backend libraries"if (-Not (get-command -ErrorAction silent ninja)) {@@ -129,6 +181,10 @@ function buildOllama() {if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}}}+}++function buildOllama() {+ mkdir -Force -path "${script:DIST_DIR}\"write-host "Building ollama CLI"& go build -trimpath -ldflags "-s -w -X=github.com/ollama/ollama/version.Version=$script:VERSION -X=github.com/ollama/ollama/server.mode=release" .if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}@@ -236,6 +292,10 @@ function distZip() {checkEnvtry {if ($($args.count) -eq 0) {+ buildCPU+ buildCUDA12+ buildCUDA13+ buildROCmbuildOllamabuildAppgatherDependencies