Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
fix(train): set the expandable_segments hint before detect_device starts CUDA (#2149) (#2156)
Yang Baifd2bda4f0209
diff --git a/changelog.d/0.76.0/2156.fixed.md b/changelog.d/0.76.0/2156.fixed.mdnew file mode 100644index 00000000..29bfc24e--- /dev/null+++ b/changelog.d/0.76.0/2156.fixed.md@@ -0,0 +1,9 @@+- **A streamed `soup train` on Linux (WSL2 included) now gets the `expandable_segments` allocator+ hint (#2149 by @ybai08 in #2156).** The hint is set only while CUDA is not yet initialised,+ and `soup train` initialised CUDA when it detected the device, before the streamed setup set+ the hint, so every streamed CUDA run printed "expandable_segments allocator hint not enabled:+ CUDA was already initialised before Soup could set it". `soup train` now sets+ `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` before it detects the device, for streamed+ runs on Linux. A value you exported is kept, and if that value is why the hint is not on, the+ line now says `PYTORCH_CUDA_ALLOC_CONF is set to something else`. Runs without+ `stream_layers`, macOS and Windows are unchanged.diff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.pyindex 8a30de9b..68eaf9c4 100644--- a/src/soup_cli/commands/train.py+++ b/src/soup_cli/commands/train.py@@ -26,6 +26,7 @@ from soup_cli.data.loader import (from soup_cli.monitoring.display import TrainingDisplayfrom soup_cli.trainer.classifier import CLASSIFICATION_TASKSfrom soup_cli.utils.gpu import (+ configure_cuda_alloc_hint,configure_mps_async_load,detect_device,get_gpu_info,@@ -1490,6 +1491,9 @@ def train().items():os.environ.setdefault(key, val)+ # Before detect_device, which creates the CUDA context the allocator reads it at (#2149).+ configure_cuda_alloc_hint(cfg.training.stream_layers)+# Detect hardware with backend awarenessdevice, device_name = detect_device(backend=cfg.backend)gpu_info = get_gpu_info(backend=cfg.backend)diff --git a/src/soup_cli/utils/gpu.py b/src/soup_cli/utils/gpu.pyindex 465efc72..3fcf0caa 100644--- a/src/soup_cli/utils/gpu.py+++ b/src/soup_cli/utils/gpu.py@@ -6,6 +6,7 @@ import jsonimport mathimport osimport re+import sysfrom typing import TYPE_CHECKING, Optionalif TYPE_CHECKING:@@ -221,6 +222,23 @@ def detect_device(backend: Optional[str] = None) -> tuple[str, str]:return "cpu", "CPU (no GPU detected)"+def configure_cuda_alloc_hint(stream_layers: bool) -> None:+ """Set PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True for a streamed run (#2149).++ The CUDA allocator reads the variable once, when the context is created, and+ ``detect_device()`` creates it, so ``soup train`` calls this first. By the time+ ``expandable_segments_status()`` runs, in the streamed setup, it is too late to+ set it there. Linux only (WSL2 included): Windows ignores the hint and macOS has+ no CUDA, so their environment is left as it is. A CPU-only Linux box gets it too,+ harmlessly; checking for a card here would start CUDA, which is the bug. A value+ the user exported is kept; an empty one counts as unset. No torch import: setting a+ variable does not need it.+ """+ if stream_layers and sys.platform.startswith("linux"):+ if not os.environ.get("PYTORCH_CUDA_ALLOC_CONF"):+ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"++def configure_mps_async_load(device: Optional[str] = None) -> None:"""Set HF_DEACTIVATE_ASYNC_LOAD=1 by default when running on Apple Silicon MPS (#2076).diff --git a/src/soup_cli/utils/layer_stream_runtime.py b/src/soup_cli/utils/layer_stream_runtime.pyindex 11ae8dec..f35ac133 100644--- a/src/soup_cli/utils/layer_stream_runtime.py+++ b/src/soup_cli/utils/layer_stream_runtime.py@@ -3026,9 +3026,13 @@ def expandable_segments_status() -> tuple[bool, str]:if not torch.cuda.is_available():return (False, "no CUDA device")if torch.cuda.is_initialized():- already = "expandable_segments:True" in os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")- if already:+ conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")+ if "expandable_segments:True" in conf:return (True, "")+ if conf:+ # ``soup train`` sets the hint before CUDA starts (#2149), with setdefault,+ # so a value the user exported is what kept it out, not the ordering.+ return (False, "PYTORCH_CUDA_ALLOC_CONF is set to something else")return (False,"CUDA was already initialised before Soup could set it — the "diff --git a/tests/test_cuda_alloc_hint.py b/tests/test_cuda_alloc_hint.pynew file mode 100644index 00000000..35c33e06--- /dev/null+++ b/tests/test_cuda_alloc_hint.py@@ -0,0 +1,109 @@+"""#2149: ``soup train`` sets the expandable_segments hint before anything starts CUDA.++``expandable_segments_status()`` only writes ``PYTORCH_CUDA_ALLOC_CONF`` while CUDA is+not initialised, and on the ``soup train`` path ``detect_device()`` initialised it long+before the streamed setup got there, so the hint never took on a CUDA run. The hint is+now set from ``train.py`` before ``detect_device()``, for streamed runs on Linux,+without overwriting a value the user exported. macOS and Windows are left alone.+"""++from __future__ import annotations++import os+import sys++import pytest+from typer.testing import CliRunner++from soup_cli.cli import app++HINT = "expandable_segments:True"+++class _StopHereError(Exception):+ """Raised by the recording ``detect_device``: nothing after it matters here."""++[email protected](autouse=True)+def clean_alloc_conf(monkeypatch: pytest.MonkeyPatch) -> None:+ # delenv on an absent key registers no undo, so set a sentinel first: teardown+ # then restores the unset state even after the code under test set it.+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "sentinel_for_teardown")+ monkeypatch.delenv("PYTORCH_CUDA_ALLOC_CONF")+++def _value_at_detect_device(monkeypatch, tmp_path, *, stream_layers: bool) -> list:+ config = tmp_path / "soup.yaml"+ config.write_text(+ "base: owner/model\n"+ "task: sft\n"+ "data:\n"+ " train: dummy.jsonl\n"+ "training:\n"+ " quantization: none\n"+ " batch_size: 1\n"+ f" stream_layers: {'true' if stream_layers else 'false'}\n",+ encoding="utf-8",+ )+ (tmp_path / "dummy.jsonl").write_text('{"text": "hi"}\n', encoding="utf-8")+ seen: list = []++ def recording_detect_device(**kwargs):+ seen.append(os.environ.get("PYTORCH_CUDA_ALLOC_CONF"))+ raise _StopHereError++ monkeypatch.setattr("soup_cli.commands.train.detect_device", recording_detect_device)+ result = CliRunner().invoke(app, ["train", "--config", str(config), "--dry-run"])+ assert isinstance(result.exception, _StopHereError), (result.output, repr(result.exception))+ return seen+++def test_a_streamed_run_on_linux_has_the_hint_before_cuda_starts(monkeypatch, tmp_path):+ monkeypatch.setattr(sys, "platform", "linux")++ seen = _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True)++ assert len(seen) == 1 and seen[0] is not None and HINT in seen[0]+++def test_a_run_that_does_not_stream_is_left_alone(monkeypatch, tmp_path):+ monkeypatch.setattr(sys, "platform", "linux")++ assert _value_at_detect_device(monkeypatch, tmp_path, stream_layers=False) == [None]++[email protected]("platform", ["win32", "darwin"])+def test_windows_and_macos_are_left_alone(monkeypatch, tmp_path, platform):+ monkeypatch.setattr(sys, "platform", platform)++ assert _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True) == [None]+++def test_a_value_the_user_exported_is_kept(monkeypatch, tmp_path):+ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:128")++ seen = _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True)++ assert seen == ["max_split_size_mb:128"]+++def test_an_empty_export_counts_as_unset(monkeypatch, tmp_path):+ """``setdefault`` would keep ``""``: the hint stays off and the old reason comes back."""+ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "")++ seen = _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True)++ assert seen == [HINT]+++def test_the_helper_does_not_import_torch(monkeypatch):+ from soup_cli.utils.gpu import configure_cuda_alloc_hint++ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.setitem(sys.modules, "torch", None) # any `import torch` now raises++ configure_cuda_alloc_hint(stream_layers=True)++ assert os.environ.get("PYTORCH_CUDA_ALLOC_CONF") == HINTdiff --git a/tests/test_issue390_expandable_segments_reason.py b/tests/test_issue390_expandable_segments_reason.pyindex 5e99e5ec..b8461200 100644--- a/tests/test_issue390_expandable_segments_reason.py+++ b/tests/test_issue390_expandable_segments_reason.py@@ -96,6 +96,19 @@ class TestExpandableSegmentsStatus:assert expandable_segments_status() == (True, "")+ def test_another_exported_value_is_named_as_the_cause(self, fake_cuda, monkeypatch):+ """#2149: once ``soup train`` sets the hint before CUDA starts, a user's own value+ is the only way to reach this branch, and "already initialised" is not why."""+ from soup_cli.utils.layer_stream_runtime import expandable_segments_status++ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:128")+ fake_cuda(available=True, initialized=True)++ enabled, why = expandable_segments_status()+ assert enabled is False+ assert why == "PYTORCH_CUDA_ALLOC_CONF is set to something else"+def test_no_cuda_says_no_cuda(self, fake_cuda, monkeypatch):from soup_cli.utils.layer_stream_runtime import expandable_segments_status