Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(dev): make the CUDA install task use the locked PyTorch (#6599)

Dmytro Mishkin5310f25893bb
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 409270ab8..5684cf404 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -100,7 +100,7 @@ The rest of this guide contains practical and technical guidance so nobody has t
pixi install -e py313 # Python 3.13
pixi run -e py313 install
- # For CUDA development (requires reinstall of PyTorch)
+ # For CUDA development
pixi run -e cuda install
```
@@ -164,13 +164,20 @@ The rest of this guide contains practical and technical guidance so nobody has t
**Dependencies:** Defined in `pyproject.toml`. Update it and run `pixi run -e default install`.
- **CUDA:** Run `pixi run -e cuda install` to install the CUDA 12.1 PyTorch build into the project `.venv`,
- which is also used by `pixi run -e cuda test-cuda`. That index stops at torch 2.5.1, the oldest version
- Kornia supports, so CUDA runs test the floor rather than the locked PyTorch. Pixi's `.pixi/envs/cuda`
- supplies the tools, not the Python dependencies under test. Commands run with `-e cuda` disable automatic uv
- synchronization so the CUDA build is preserved. Re-run the CUDA install task after changing dependencies or
- using a default-environment command that synchronizes `.venv`. For focused CUDA tests, use
- `pixi run -e cuda test-module tests/<path> --device=cuda`.
+ **CUDA:** Run `pixi run -e cuda install` to install the locked PyTorch as a CUDA build into the project `.venv`,
+ which is also used by `pixi run -e cuda test-cuda`. PyPI's Linux wheels are already CUDA builds, so on Linux the
+ task only syncs the lock, reinstalling every package so that removing a stale wheel cannot take files a locked
+ wheel needs. PyPI's Windows wheels are CPU-only, so on Windows the task then replaces torch with the same version
+ from the PyTorch CUDA index; a lock update to a new PyTorch has to update that version and index in `pixi.toml`
+ and the index in `pyproject.toml` too. Pixi's `.pixi/envs/cuda` supplies the tools, not the Python dependencies
+ under test. Commands run with `-e cuda` do not synchronize `.venv` automatically, so re-run the CUDA install task
+ after a dependency or lock change. On Windows this is what preserves the CUDA build, and a default-environment
+ command that synchronizes `.venv` replaces it, so re-run the task after one of those too. For focused CUDA tests,
+ use `pixi run -e cuda test-module tests/<path> --device=cuda`.
+
+ The locked build needs an NVIDIA driver from the R580 branch or newer and a Turing or newer GPU. Check
+ `torch.cuda.is_available()` after installing: when it is false, a `--device=cuda` run skips its tests and still
+ exits 0.
5. **Develop and test:**
diff --git a/changelog.d/6599.fixed.md b/changelog.d/6599.fixed.md
new file mode 100644
index 000000000..24f52cafc
--- /dev/null
+++ b/changelog.d/6599.fixed.md
@@ -0,0 +1,6 @@
+`pixi run -e cuda install` now installs the locked PyTorch as a CUDA build. It used to reinstall torch from the CUDA
+12.1 index, which stops at torch 2.5.1, so a CUDA development environment silently ran the oldest supported PyTorch and
+moved several locked dependencies, such as triton and sympy, to other versions. The locked build needs an NVIDIA driver
+from the R580 branch or newer and a Turing or newer GPU. Re-run this task on a `.venv` the old one created. Until then,
+default-environment commands leave `import torch` failing: on `libcudnn.so.9` after `pixi run -e default install`, and
+on `undefined symbol: ncclCommResume` after a `uv run` task such as `pixi run test-module`.
diff --git a/pixi.toml b/pixi.toml
index 8380c3968..d24d25cca 100644
--- a/pixi.toml
+++ b/pixi.toml
@@ -74,23 +74,33 @@ platforms = ["linux-64", "linux-aarch64", "win-64"]
# ------------------------------------------------------------------------------
# Feature: CUDA (PyTorch CUDA wheels) - for local GPU development
-# Requires reinstall because uv doesn't support per-environment sources
+# PyPI's Linux wheels, which uv.lock selects, are already CUDA builds. Its Windows wheels are CPU-only, and uv
+# doesn't support per-environment sources, so the Windows tasks replace torch after syncing.
# ------------------------------------------------------------------------------
[feature.cuda.activation.env]
-# Preserve the CUDA wheel override when running tests or other project commands.
+# On Windows, preserve the CUDA wheel that replaces the locked torch when running tests or other project commands.
+# Linux has no such wheel, but this applies there too: re-run the install task after a lock change.
UV_NO_SYNC = "1"
[feature.cuda.tasks]
-# Run pip inside uv's project environment, not Pixi's activated Conda environment.
-install = { cmd = "uv sync --extra dev && uv run --no-sync uv pip install --reinstall torch --index https://download.pytorch.org/whl/cu121", description = "Install dev dependencies (CUDA PyTorch)" }
-install-docs = { cmd = "uv sync --extra dev --extra docs && uv run --no-sync uv pip install --reinstall torch --index https://download.pytorch.org/whl/cu121", description = "Install dev + docs (CUDA PyTorch)" }
+# --reinstall restores what removing a stale wheel deletes: the nvidia-*-cu12 wheels of an older install own paths the
+# locked nvidia-*-cu13 wheels also own, and after a plain sync `import torch` fails on libcudnn.so.9.
+install = { cmd = "uv sync --extra dev --reinstall", description = "Install dev dependencies (CUDA PyTorch)" }
+install-docs = { cmd = "uv sync --extra dev --extra docs --reinstall", description = "Install dev + docs (CUDA PyTorch)" }
# CUDA test tasks
test-cuda = { cmd = "uv run pytest tests/", env = { KORNIA_TEST_DEVICE = "cuda" }, description = "Run tests on CUDA" }
test-cuda-f32 = { cmd = "uv run pytest tests/", env = { KORNIA_TEST_DEVICE = "cuda", KORNIA_TEST_DTYPE = "float32" } }
test-cuda-f64 = { cmd = "uv run pytest tests/", env = { KORNIA_TEST_DEVICE = "cuda", KORNIA_TEST_DTYPE = "float64" } }
test-cuda-half = { cmd = "uv run pytest tests/ --isolate-half-precision", env = { KORNIA_TEST_DEVICE = "cuda", KORNIA_TEST_DTYPE = "float16,bfloat16" }, description = "Run half-precision CUDA tests with per-test subprocess isolation" }
+[feature.cuda.target.win-64.tasks]
+# Run pip inside uv's project environment, not Pixi's activated Conda environment. Keep the version equal to torch in
+# uv.lock and the index matching its cuda-toolkit, and --no-deps so the index's dependencies do not replace the locked
+# ones; tests/test_pixi_tasks.py covers all three.
+install = { cmd = "uv sync --extra dev && uv run --no-sync uv pip install --reinstall --no-deps torch==2.15.0 --index https://download.pytorch.org/whl/cu132", description = "Install dev dependencies (CUDA PyTorch)" }
+install-docs = { cmd = "uv sync --extra dev --extra docs && uv run --no-sync uv pip install --reinstall --no-deps torch==2.15.0 --index https://download.pytorch.org/whl/cu132", description = "Install dev + docs (CUDA PyTorch)" }
+
# ------------------------------------------------------------------------------
# Python version features (for CI matrix)
# ------------------------------------------------------------------------------
diff --git a/pyproject.toml b/pyproject.toml
index 948693473..495a5070c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -76,7 +76,8 @@ build-backend = "hatchling.build"
# uv configuration
# =============================================================================
# PyTorch: Use standard PyPI version (simpler, more compatible)
-# For CUDA: pixi run -e cuda install (uses reinstall workaround)
+# For CUDA: pixi run -e cuda install
+# PyPI's Linux wheels are CUDA builds; on Windows that task reinstalls torch from the index below
# Note: uv doesn't support per-environment sources
# torch is installed from standard PyPI (no custom index needed)
@@ -450,11 +451,6 @@ invalid-assignment = "ignore"
redundant-cast = "ignore"
[[tool.uv.index]]
-name = "pytorch-cu121"
-url = "https://download.pytorch.org/whl/cu121"
-explicit = true
-
-[[tool.uv.index]]
-name = "pytorch-cu124"
-url = "https://download.pytorch.org/whl/cu124"
+name = "pytorch-cu132"
+url = "https://download.pytorch.org/whl/cu132"
explicit = true
diff --git a/tests/test_pixi_tasks.py b/tests/test_pixi_tasks.py
index 38fd416d2..7637d3af0 100644
--- a/tests/test_pixi_tasks.py
+++ b/tests/test_pixi_tasks.py
@@ -24,6 +24,7 @@ import shutil
import subprocess
import sys
import tomllib
+from functools import partial
from pathlib import Path
from zipfile import ZipFile
@@ -31,65 +32,122 @@ import pytest
ROOT = Path(__file__).resolve().parents[1]
PIXI = tomllib.loads((ROOT / "pixi.toml").read_text(encoding="utf-8"))
+LOCKED_PACKAGES = tomllib.loads((ROOT / "uv.lock").read_text(encoding="utf-8"))["package"]
+LOCKED_TORCH = next(package for package in LOCKED_PACKAGES if package["name"] == "torch")
+LOCKED_CUDA_TOOLKIT = next(package for package in LOCKED_PACKAGES if package["name"] == "cuda-toolkit")
+# PyPI's Linux wheels are the build of this toolkit, so toolkit 13.2 means the cu132 index.
+WINDOWS_CUDA_BUILD = "cu" + "".join(LOCKED_CUDA_TOOLKIT["version"].split(".")[:2])
pytestmark = pytest.mark.device_agnostic
[email protected]("task", ["install", "install-docs"])
-def test_cuda_index_is_a_url(task: str) -> None:
- args = shlex.split(PIXI["feature"]["cuda"]["tasks"][task]["cmd"])
- project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
- cuda_index = next(index for index in project["tool"]["uv"]["index"] if index["name"] == "pytorch-cu121")
- # --index takes a URL, not the configured index's name.
- assert args[args.index("--index") + 1] == cuda_index["url"]
+def _cuda_tasks(platform: str) -> dict[str, dict[str, str]]:
+ # A target's tasks replace the feature's tasks of the same name on that platform, as Pixi resolves them.
+ cuda = PIXI["feature"]["cuda"]
+ return cuda["tasks"] | cuda.get("target", {}).get(platform, {}).get("tasks", {})
[email protected]("task", ["install", "install-docs"])
[email protected]("activation", ["CONDA_PREFIX", "VIRTUAL_ENV"])
-def test_cuda_install_and_run_share_environment(tmp_path: Path, task: str, activation: str) -> None:
- uv = shutil.which("uv")
- if uv is None:
- pytest.skip("uv is required; run this test through Pixi")
+def _cuda_uv_env(tmp_path: Path, uv: str) -> dict[str, str]:
env = {key: value for key, value in os.environ.items() if not key.startswith("UV_")}
for key in ("CONDA_PREFIX", "CONDA_DEFAULT_ENV", "VIRTUAL_ENV"):
env.pop(key, None)
- cuda = PIXI["feature"]["cuda"]
# Activation of every feature the cuda environment composes, as Pixi applies it.
env.update(PIXI.get("activation", {}).get("env", {}))
for feature in PIXI["environments"]["cuda"]["features"]:
env.update(PIXI["feature"][feature].get("activation", {}).get("env", {}))
- project_env = env.get("UV_PROJECT_ENVIRONMENT", ".venv")
env.update(UV_CACHE_DIR=str(tmp_path / "cache"), UV_OFFLINE="1", UV_PYTHON_DOWNLOADS="never")
env["PATH"] = str(Path(uv).parent) + os.pathsep + env["PATH"]
+ return env
- def run(args: list[str]) -> str:
- result = subprocess.run( # noqa: S603
- args, cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30, check=False
- )
- assert result.returncode == 0, result.stdout + result.stderr
- return result.stdout
-
- # Tiny metadata-only wheels exercise real uv selection and re-sync behavior,
- # without a GPU, network access, or a PyTorch download.
- for version in ("2.14.0", "2.5.1+cu121"):
- info = f"torch-{version}.dist-info"
- files = {
- f"{info}/METADATA": f"Metadata-Version: 2.1\nName: torch\nVersion: {version}\n",
- f"{info}/WHEEL": "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n",
- }
- files[f"{info}/RECORD"] = "".join(f"{name},,\n" for name in [*files, f"{info}/RECORD"])
- with ZipFile(tmp_path / f"torch-{version}-py3-none-any.whl", "w") as wheel:
- for name, content in files.items():
- wheel.writestr(name, content)
- (tmp_path / "pyproject.toml").write_text(
+
+def _run(args: list[str], cwd: Path, env: dict[str, str]) -> str:
+ result = subprocess.run(args, cwd=cwd, env=env, capture_output=True, text=True, timeout=30, check=False) # noqa: S603
+ assert result.returncode == 0, result.stdout + result.stderr
+ return result.stdout
+
+
+def _write_wheel(
+ directory: Path, name: str, version: str, requires: str | None = None, modules: tuple[str, ...] = ()
+) -> None:
+ # A tiny wheel exercises real uv selection and re-sync behavior, without a GPU, network access, or a PyTorch
+ # download.
+ info = f"{name}-{version}.dist-info"
+ requirement = f"Requires-Dist: {requires}\n" if requires else ""
+ files = dict.fromkeys(modules, "")
+ files[f"{info}/METADATA"] = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n{requirement}"
+ files[f"{info}/WHEEL"] = "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n"
+ files[f"{info}/RECORD"] = "".join(f"{file},,\n" for file in [*files, f"{info}/RECORD"])
+ with ZipFile(directory / f"{name}-{version}-py3-none-any.whl", "w") as wheel:
+ for file, content in files.items():
+ wheel.writestr(file, content)
+
+
+def _write_probe_project(directory: Path, dependency: str, version: str) -> None:
+ (directory / "pyproject.toml").write_text(
'[project]\nname = "pixi-task-probe"\nversion = "0.0.0"\nrequires-python = ">=3.11"\n'
- 'dependencies = ["torch"]\n[project.optional-dependencies]\ndev = []\ndocs = []\n'
- '[tool.uv.sources]\ntorch = {path = "torch-2.14.0-py3-none-any.whl"}\n',
+ f'dependencies = ["{dependency}"]\n[project.optional-dependencies]\ndev = []\ndocs = []\n'
+ f'[tool.uv.sources]\n{dependency} = {{path = "{dependency}-{version}-py3-none-any.whl"}}\n',
encoding="utf-8",
)
+
+
+def test_locked_linux_torch_is_a_cuda_build() -> None:
+ # The Linux install task only syncs the lock, so it is only a CUDA install while this holds on x86-64 and aarch64.
+ assert any(
+ dependency["name"].startswith("nvidia-cudnn")
+ and "sys_platform == 'linux'" in dependency.get("marker", "")
+ and "platform_machine" not in dependency["marker"]
+ for dependency in LOCKED_TORCH["dependencies"]
+ )
+
+
[email protected]("task", ["install", "install-docs"])
+def test_cuda_index_is_a_url(task: str) -> None:
+ args = shlex.split(_cuda_tasks("win-64")[task]["cmd"])
+ project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
+ indexes = {index["name"]: index["url"] for index in project["tool"]["uv"]["index"]}
+ # --index takes a URL, not the configured index's name. It also has to be the build of the locked cuda-toolkit, so
+ # a lock update fails here until the index follows it.
+ assert args[args.index("--index") + 1] == indexes.get(f"pytorch-{WINDOWS_CUDA_BUILD}")
+
+
+ ("platform", "expected_torch"),
+ [
+ ("linux-64", LOCKED_TORCH["version"]),
+ ("linux-aarch64", LOCKED_TORCH["version"]),
+ ("win-64", f"{LOCKED_TORCH['version']}+{WINDOWS_CUDA_BUILD}"),
+ ],
+)
[email protected]("task", ["install", "install-docs"])
[email protected]("activation", ["CONDA_PREFIX", "VIRTUAL_ENV"])
+def test_cuda_install_and_run_share_environment(
+ tmp_path: Path, task: str, activation: str, platform: str, expected_torch: str
+) -> None:
+ uv = shutil.which("uv")
+ if uv is None:
+ pytest.skip("uv is required; run this test through Pixi")
+ env = _cuda_uv_env(tmp_path, uv)
+ project_env = env.get("UV_PROJECT_ENVIRONMENT", ".venv")
+ run = partial(_run, cwd=tmp_path, env=env)
+
+ # The index build of the locked version requires a package nothing provides, as a stand-in for the index's
+ # dependencies replacing the locked ones. Its newer build must not be selected at all: an index moves on
+ # independently of the lock.
+ locked = LOCKED_TORCH["version"]
+ index_build = f"{locked}+{WINDOWS_CUDA_BUILD}"
+ newer_index_build = f"99.0.0+{WINDOWS_CUDA_BUILD}"
+ _write_wheel(tmp_path, "torch", locked)
+ _write_wheel(tmp_path, "torch", index_build, requires="pixi-task-probe-not-installable")
+ _write_wheel(tmp_path, "torch", newer_index_build)
+ _write_probe_project(tmp_path, "torch", locked)
index = tmp_path / "simple" / "torch"
index.mkdir(parents=True)
(index / "index.html").write_text(
- '<a href="../../torch-2.5.1%2Bcu121-py3-none-any.whl">torch</a>', encoding="utf-8"
+ "".join(
+ f'<a href="../../torch-{version.replace("+", "%2B")}-py3-none-any.whl">torch</a>\n'
+ for version in (index_build, newer_index_build)
+ ),
+ encoding="utf-8",
)
pixi_env = tmp_path / ".pixi" / "envs" / "cuda"
run([uv, "venv", "--python", sys.executable, str(pixi_env)])
@@ -101,7 +159,7 @@ def test_cuda_install_and_run_share_environment(tmp_path: Path, task: str, activ
env[activation] = str(pixi_env)
if activation == "CONDA_PREFIX":
env["CONDA_DEFAULT_ENV"] = "kornia:cuda" # as `pixi run -e cuda` sets it
- for command in cuda["tasks"][task]["cmd"].split(" && "):
+ for command in _cuda_tasks(platform)[task]["cmd"].split(" && "):
args = shlex.split(command)
if "--index" in args:
args[args.index("--index") + 1] = index.parent.as_uri()
@@ -110,16 +168,42 @@ def test_cuda_install_and_run_share_environment(tmp_path: Path, task: str, activ
# Replace only pytest with a metadata probe; preserve every uv run option
# from the CUDA test tasks and the inherited focused-test task.
probe = "import json,sys; from importlib.metadata import version; print(json.dumps([sys.prefix,version('torch')]))"
- tasks = PIXI["tasks"] | cuda["tasks"]
+ tasks = PIXI["tasks"] | _cuda_tasks(platform)
for name in ("test-cuda", "test-cuda-f32", "test-cuda-f64", "test-cuda-half", "test-module"):
args = shlex.split(tasks[name]["cmd"])
env.update(tasks[name].get("env", {}))
prefix, version = json.loads(run([uv, *args[1 : args.index("pytest")], "python", "-c", probe]))
assert Path(prefix) == tmp_path / project_env
- assert version == "2.5.1+cu121"
+ assert version == expected_torch
assert json.loads(run([uv, "pip", "list", "--python", str(pixi_env), "--format", "json"])) == []
[email protected]("task", ["install", "install-docs"])
+def test_cuda_install_restores_files_shared_with_a_stale_wheel(tmp_path: Path, task: str) -> None:
+ # A wheel left by an older install can own paths a locked wheel also owns, as nvidia-*-cu12 and nvidia-*-cu13 do.
+ # Removing the stale wheel deletes those files while uv still counts the locked wheel as installed, and
+ # `import torch` then fails on libcudnn.so.9.
+ uv = shutil.which("uv")
+ if uv is None:
+ pytest.skip("uv is required; run this test through Pixi")
+ env = _cuda_uv_env(tmp_path, uv)
+ project_env = env.get("UV_PROJECT_ENVIRONMENT", ".venv")
+ run = partial(_run, cwd=tmp_path, env=env)
+ _write_wheel(tmp_path, "lockedlibs", "1.0.0", modules=("shared_library.py",))
+ _write_wheel(tmp_path, "stalelibs", "1.0.0", modules=("shared_library.py",))
+ _write_probe_project(tmp_path, "lockedlibs", "1.0.0")
+ run([uv, "venv", "--python", sys.executable, project_env])
+ install = [[uv, *shlex.split(command)[1:]] for command in _cuda_tasks("linux-64")[task]["cmd"].split(" && ")]
+
+ for args in install:
+ run(args)
+ run([uv, "pip", "install", "--python", project_env, "stalelibs-1.0.0-py3-none-any.whl"])
+ for args in install:
+ run(args)
+
+ run([uv, "run", "--no-sync", "python", "-c", "import shared_library"])
+
+
@pytest.mark.parametrize("environment", ["default", "py311", "py312", "py313", "py314"])
def test_non_cuda_environment_selection(environment: str) -> None:
env = dict(PIXI.get("activation", {}).get("env", {}))