Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Add Jetson cuda variants for arm
Daniel Hiltgend470ebe78bc7
diff --git a/Dockerfile b/Dockerfileindex 8eb90057f..79b2a6961 100644--- a/Dockerfile+++ b/Dockerfile@@ -3,6 +3,9 @@ ARG CMAKE_VERSION=3.22.1# this CUDA_VERSION corresponds with the one specified in docs/gpu.mdARG CUDA_VERSION=11.3.1ARG ROCM_VERSION=6.1.2+ARG JETPACK_6=r36.2.0+ARG JETPACK_5=r35.4.1+ARG JETPACK_4=r32.7.1# Copy the minimal context we need to run the generate scriptsFROM scratch AS llm-code@@ -22,7 +25,7 @@ ENV GOARCH amd64RUN --mount=type=cache,target=/root/.ccache \OLLAMA_SKIP_STATIC_GENERATE=1 OLLAMA_SKIP_CPU_GENERATE=1 bash gen_linux.sh-FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION-devel-rockylinux8 AS cuda-build-arm64+FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION-devel-rockylinux8 AS cuda-build-server-arm64ARG CMAKE_VERSIONCOPY ./scripts/rh_linux_deps.sh /RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh@@ -31,11 +34,40 @@ COPY --from=llm-code / /go/src/github.com/ollama/ollama/WORKDIR /go/src/github.com/ollama/ollama/llm/generateARG CGO_CFLAGSENV GOARCH arm64+RUN OLLAMA_SKIP_STATIC_GENERATE=1 OLLAMA_SKIP_CPU_GENERATE=1 bash gen_linux.sh++FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK_6} AS cuda-build-jetpack6-arm64+ARG CMAKE_VERSION+RUN apt-get update && apt-get install -y git curl && \+ curl -s -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz | tar -zx -C /usr --strip-components 1+COPY --from=llm-code / /go/src/github.com/ollama/ollama/+WORKDIR /go/src/github.com/ollama/ollama/llm/generate+ARG CGO_CFLAGS+ENV GOARCH arm64+ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs+RUN --mount=type=cache,target=/root/.ccache \+ OLLAMA_SKIP_STATIC_GENERATE=1 \+ OLLAMA_SKIP_CPU_GENERATE=1 \+ CUDA_VARIANT="_jetpack6" \+ CUDA_DIST_DIR="/go/src/github.com/ollama/ollama/dist/linux-arm64/ollama_libs/cuda_jetpack6" \+ CMAKE_CUDA_ARCHITECTURES="87" \+ bash gen_linux.sh++FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK_5} AS cuda-build-jetpack5-arm64+ARG CMAKE_VERSION+RUN apt-get update && apt-get install -y git curl && \+ curl -s -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz | tar -zx -C /usr --strip-components 1+COPY --from=llm-code / /go/src/github.com/ollama/ollama/+WORKDIR /go/src/github.com/ollama/ollama/llm/generate+ARG CGO_CFLAGS+ENV GOARCH arm64+ENV LIBRARY_PATH /usr/local/cuda/lib64/stubsRUN --mount=type=cache,target=/root/.ccache \OLLAMA_SKIP_STATIC_GENERATE=1 \OLLAMA_SKIP_CPU_GENERATE=1 \- CMAKE_CUDA_ARCHITECTURES="${CUDA_V11_ARCHITECTURES}" \- CUDA_VARIANT="_v11" \+ CUDA_VARIANT="_jetpack5" \+ CUDA_DIST_DIR="/go/src/github.com/ollama/ollama/dist/linux-arm64/ollama_libs/cuda_jetpack5" \+ CMAKE_CUDA_ARCHITECTURES="72;87" \bash gen_linux.shFROM --platform=linux/amd64 rocm/dev-centos-7:${ROCM_VERSION}-complete AS rocm-build-amd64@@ -123,8 +155,14 @@ ARG GOLANG_VERSIONWORKDIR /go/src/github.com/ollama/ollamaCOPY . .COPY --from=static-build-arm64 /go/src/github.com/ollama/ollama/llm/build/linux/ llm/build/linux/-COPY --from=cuda-build-arm64 /go/src/github.com/ollama/ollama/dist/ dist/-COPY --from=cuda-build-arm64 /go/src/github.com/ollama/ollama/llm/build/linux/ llm/build/linux/+COPY --from=cuda-build-server-arm64 /go/src/github.com/ollama/ollama/dist/ dist/+COPY --from=cuda-build-server-arm64 /go/src/github.com/ollama/ollama/llm/build/linux/ llm/build/linux/+## arm binary += 381M+COPY --from=cuda-build-jetpack6-arm64 /go/src/github.com/ollama/ollama/llm/build/linux/ llm/build/linux/+COPY --from=cuda-build-jetpack6-arm64 /go/src/github.com/ollama/ollama/dist/ dist/+## arm binary += 330M+COPY --from=cuda-build-jetpack5-arm64 /go/src/github.com/ollama/ollama/llm/build/linux/ llm/build/linux/+COPY --from=cuda-build-jetpack5-arm64 /go/src/github.com/ollama/ollama/dist/ dist/ARG GOFLAGSARG CGO_CFLAGSRUN --mount=type=cache,target=/root/.ccache \diff --git a/gpu/gpu.go b/gpu/gpu.goindex d0ae0f34f..22461922d 100644--- a/gpu/gpu.go+++ b/gpu/gpu.go@@ -15,7 +15,9 @@ import ("log/slog""os""path/filepath"+ "regexp""runtime"+ "strconv""strings""sync""unsafe"@@ -215,7 +217,7 @@ func GetGPUInfo() GpuInfoList {GpuInfo: GpuInfo{memInfo: mem,Library: "cpu",- Variant: cpuCapability,+ Variant: cpuCapability.String(),ID: "0",},},@@ -231,6 +233,35 @@ func GetGPUInfo() GpuInfoList {depPath := GetDepDir()+ var cudaVariant string+ if runtime.GOARCH == "arm64" && runtime.GOOS == "linux" {+ if CudaTegra != "" {+ ver := strings.Split(CudaTegra, ".")+ if len(ver) > 0 {+ cudaVariant = "jetpack" + ver[0]+ }+ } else if data, err := os.ReadFile("/etc/nv_tegra_release"); err == nil {+ r := regexp.MustCompile(` R(\d+) `)+ m := r.FindSubmatch(data)+ if len(m) != 2 {+ slog.Info("Unexpected format for /etc/nv_tegra_release. Set JETSON_JETPACK to select version")+ } else {+ if l4t, err := strconv.Atoi(string(m[1])); err == nil {+ // Note: mapping from L4t -> JP is inconsistent (can't just subtract 30)+ // https://developer.nvidia.com/embedded/jetpack-archive+ switch l4t {+ case 35:+ cudaVariant = "jetpack5"+ case 36:+ cudaVariant = "jetpack6"+ default:+ slog.Info("unsupported L4T version", "nv_tegra_release", string(data))+ }+ }+ }+ }+ }+// Load ALL librariescHandles = initCudaHandles()@@ -240,6 +271,7 @@ func GetGPUInfo() GpuInfoList {gpuInfo := CudaGPUInfo{GpuInfo: GpuInfo{Library: "cuda",+ Variant: cudaVariant,},index: i,}@@ -266,7 +298,15 @@ func GetGPUInfo() GpuInfoList {gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)gpuInfo.MinimumMemory = cudaMinimumMemory- gpuInfo.DependencyPath = depPath+ if depPath != "" {+ gpuInfo.DependencyPath = depPath+ // Check for variant specific directory+ if cudaVariant != "" {+ if _, err := os.Stat(filepath.Join(depPath, "cuda_"+cudaVariant)); err == nil {+ gpuInfo.DependencyPath = filepath.Join(depPath, "cuda_"+cudaVariant)+ }+ }+ }gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])gpuInfo.DriverMajor = driverMajorgpuInfo.DriverMinor = driverMinordiff --git a/gpu/gpu_darwin.go b/gpu/gpu_darwin.goindex 9d9fd84eb..417b48dfa 100644--- a/gpu/gpu_darwin.go+++ b/gpu/gpu_darwin.go@@ -25,7 +25,7 @@ func GetGPUInfo() GpuInfoList {return []GpuInfo{{Library: "cpu",- Variant: GetCPUCapability(),+ Variant: GetCPUCapability().String(),memInfo: mem,},}@@ -48,7 +48,7 @@ func GetCPUInfo() GpuInfoList {return []GpuInfo{{Library: "cpu",- Variant: GetCPUCapability(),+ Variant: GetCPUCapability().String(),memInfo: mem,},}diff --git a/gpu/types.go b/gpu/types.goindex 8d22b06bf..fc628d471 100644--- a/gpu/types.go+++ b/gpu/types.go@@ -19,7 +19,7 @@ type GpuInfo struct {Library string `json:"library,omitempty"`// Optional variant to select (e.g. versions, cpu feature flags)- Variant CPUCapability `json:"variant"`+ Variant string `json:"variant"`// MinimumMemory represents the minimum memory required to use the GPUMinimumMemory uint64 `json:"-"`@@ -81,8 +81,8 @@ func (l GpuInfoList) ByLibrary() []GpuInfoList {for _, info := range l {found := falserequested := info.Library- if info.Variant != CPUCapabilityNone {- requested += "_" + info.Variant.String()+ if info.Variant != CPUCapabilityNone.String() {+ requested += "_" + info.Variant}for i, lib := range libs {if lib == requested {diff --git a/llm/generate/gen_linux.sh b/llm/generate/gen_linux.shindex 1365d07db..dc9dda5a3 100755--- a/llm/generate/gen_linux.sh+++ b/llm/generate/gen_linux.sh@@ -165,7 +165,7 @@ if [ -z "${OLLAMA_SKIP_CUDA_GENERATE}" -a -d "${CUDA_LIB_DIR}" ]; thenecho "CUDA libraries detected - building dynamic CUDA library"init_varsCUDA_MAJOR=$(ls "${CUDA_LIB_DIR}"/libcudart.so.* | head -1 | cut -f3 -d. || true)- if [ -n "${CUDA_MAJOR}" ]; then+ if [ -n "${CUDA_MAJOR}" -a -z "${CUDA_VARIANT}" ]; thenCUDA_VARIANT=_v${CUDA_MAJOR}fiif [ "${ARCH}" == "arm64" ]; then@@ -189,9 +189,10 @@ if [ -z "${OLLAMA_SKIP_CUDA_GENERATE}" -a -d "${CUDA_LIB_DIR}" ]; thenCMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} ${ARM64_DEFS} ${CMAKE_CUDA_DEFS} -DGGML_STATIC=off"BUILD_DIR="../build/linux/${ARCH}/cuda${CUDA_VARIANT}"export LLAMA_SERVER_LDFLAGS="-L${CUDA_LIB_DIR} -lcudart -lcublas -lcublasLt -lcuda"- CUDA_DIST_DIR="${DIST_BASE}/ollama_libs"+ CUDA_DIST_DIR="${CUDA_DIST_DIR:-${DIST_BASE}/ollama_libs}"buildinstall+ echo "Installing CUDA dependencies in ${CUDA_DIST_DIR}"mkdir -p "${CUDA_DIST_DIR}"for lib in ${CUDA_LIB_DIR}/libcudart.so* ${CUDA_LIB_DIR}/libcublas.so* ${CUDA_LIB_DIR}/libcublasLt.so* ; docp -a "${lib}" "${CUDA_DIST_DIR}"diff --git a/llm/payload.go b/llm/payload.goindex b402e1f24..963b32959 100644--- a/llm/payload.go+++ b/llm/payload.go@@ -82,8 +82,8 @@ func serversForGpu(info gpu.GpuInfo) []string {// glob workDir for files that start with ollama_availableServers := getAvailableServers()requested := info.Library- if info.Variant != gpu.CPUCapabilityNone {- requested += "_" + info.Variant.String()+ if info.Variant != gpu.CPUCapabilityNone.String() {+ requested += "_" + info.Variant}servers := []string{}diff --git a/scripts/build_linux.sh b/scripts/build_linux.shindex ebb60c5a7..adda2ad75 100755--- a/scripts/build_linux.sh+++ b/scripts/build_linux.sh@@ -22,6 +22,7 @@ for TARGETARCH in ${BUILD_ARCH}; do-t builder:$TARGETARCH \.docker create --platform linux/$TARGETARCH --name builder-$TARGETARCH builder:$TARGETARCH+ rm -rf ./dist/linux-$TARGETARCHdocker cp builder-$TARGETARCH:/go/src/github.com/ollama/ollama/dist/linux-$TARGETARCH ./distdocker rm builder-$TARGETARCHecho "Compressing final linux bundle..."