Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(train): set the expandable_segments hint before detect_device starts CUDA (#2149) (#2156)

Yang Baifd2bda4f0209
diff --git a/changelog.d/0.76.0/2156.fixed.md b/changelog.d/0.76.0/2156.fixed.md
new file mode 100644
index 00000000..29bfc24e
--- /dev/null
+++ b/changelog.d/0.76.0/2156.fixed.md
@@ -0,0 +1,9 @@
+- **A streamed `soup train` on Linux (WSL2 included) now gets the `expandable_segments` allocator
+ hint (#2149 by @ybai08 in #2156).** The hint is set only while CUDA is not yet initialised,
+ and `soup train` initialised CUDA when it detected the device, before the streamed setup set
+ the hint, so every streamed CUDA run printed "expandable_segments allocator hint not enabled:
+ CUDA was already initialised before Soup could set it". `soup train` now sets
+ `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` before it detects the device, for streamed
+ runs on Linux. A value you exported is kept, and if that value is why the hint is not on, the
+ line now says `PYTORCH_CUDA_ALLOC_CONF is set to something else`. Runs without
+ `stream_layers`, macOS and Windows are unchanged.
diff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.py
index 8a30de9b..68eaf9c4 100644
--- a/src/soup_cli/commands/train.py
+++ b/src/soup_cli/commands/train.py
@@ -26,6 +26,7 @@ from soup_cli.data.loader import (
from soup_cli.monitoring.display import TrainingDisplay
from soup_cli.trainer.classifier import CLASSIFICATION_TASKS
from soup_cli.utils.gpu import (
+ configure_cuda_alloc_hint,
configure_mps_async_load,
detect_device,
get_gpu_info,
@@ -1490,6 +1491,9 @@ def train(
).items():
os.environ.setdefault(key, val)
+ # Before detect_device, which creates the CUDA context the allocator reads it at (#2149).
+ configure_cuda_alloc_hint(cfg.training.stream_layers)
+
# Detect hardware with backend awareness
device, device_name = detect_device(backend=cfg.backend)
gpu_info = get_gpu_info(backend=cfg.backend)
diff --git a/src/soup_cli/utils/gpu.py b/src/soup_cli/utils/gpu.py
index 465efc72..3fcf0caa 100644
--- a/src/soup_cli/utils/gpu.py
+++ b/src/soup_cli/utils/gpu.py
@@ -6,6 +6,7 @@ import json
import math
import os
import re
+import sys
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
@@ -221,6 +222,23 @@ def detect_device(backend: Optional[str] = None) -> tuple[str, str]:
return "cpu", "CPU (no GPU detected)"
+def configure_cuda_alloc_hint(stream_layers: bool) -> None:
+ """Set PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True for a streamed run (#2149).
+
+ The CUDA allocator reads the variable once, when the context is created, and
+ ``detect_device()`` creates it, so ``soup train`` calls this first. By the time
+ ``expandable_segments_status()`` runs, in the streamed setup, it is too late to
+ set it there. Linux only (WSL2 included): Windows ignores the hint and macOS has
+ no CUDA, so their environment is left as it is. A CPU-only Linux box gets it too,
+ harmlessly; checking for a card here would start CUDA, which is the bug. A value
+ the user exported is kept; an empty one counts as unset. No torch import: setting a
+ variable does not need it.
+ """
+ if stream_layers and sys.platform.startswith("linux"):
+ if not os.environ.get("PYTORCH_CUDA_ALLOC_CONF"):
+ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
+
def configure_mps_async_load(device: Optional[str] = None) -> None:
"""Set HF_DEACTIVATE_ASYNC_LOAD=1 by default when running on Apple Silicon MPS (#2076).
diff --git a/src/soup_cli/utils/layer_stream_runtime.py b/src/soup_cli/utils/layer_stream_runtime.py
index 11ae8dec..f35ac133 100644
--- a/src/soup_cli/utils/layer_stream_runtime.py
+++ b/src/soup_cli/utils/layer_stream_runtime.py
@@ -3026,9 +3026,13 @@ def expandable_segments_status() -> tuple[bool, str]:
if not torch.cuda.is_available():
return (False, "no CUDA device")
if torch.cuda.is_initialized():
- already = "expandable_segments:True" in os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
- if already:
+ conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
+ if "expandable_segments:True" in conf:
return (True, "")
+ if conf:
+ # ``soup train`` sets the hint before CUDA starts (#2149), with setdefault,
+ # so a value the user exported is what kept it out, not the ordering.
+ return (False, "PYTORCH_CUDA_ALLOC_CONF is set to something else")
return (
False,
"CUDA was already initialised before Soup could set it — the "
diff --git a/tests/test_cuda_alloc_hint.py b/tests/test_cuda_alloc_hint.py
new file mode 100644
index 00000000..35c33e06
--- /dev/null
+++ b/tests/test_cuda_alloc_hint.py
@@ -0,0 +1,109 @@
+"""#2149: ``soup train`` sets the expandable_segments hint before anything starts CUDA.
+
+``expandable_segments_status()`` only writes ``PYTORCH_CUDA_ALLOC_CONF`` while CUDA is
+not initialised, and on the ``soup train`` path ``detect_device()`` initialised it long
+before the streamed setup got there, so the hint never took on a CUDA run. The hint is
+now set from ``train.py`` before ``detect_device()``, for streamed runs on Linux,
+without overwriting a value the user exported. macOS and Windows are left alone.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import pytest
+from typer.testing import CliRunner
+
+from soup_cli.cli import app
+
+HINT = "expandable_segments:True"
+
+
+class _StopHereError(Exception):
+ """Raised by the recording ``detect_device``: nothing after it matters here."""
+
+
[email protected](autouse=True)
+def clean_alloc_conf(monkeypatch: pytest.MonkeyPatch) -> None:
+ # delenv on an absent key registers no undo, so set a sentinel first: teardown
+ # then restores the unset state even after the code under test set it.
+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "sentinel_for_teardown")
+ monkeypatch.delenv("PYTORCH_CUDA_ALLOC_CONF")
+
+
+def _value_at_detect_device(monkeypatch, tmp_path, *, stream_layers: bool) -> list:
+ config = tmp_path / "soup.yaml"
+ config.write_text(
+ "base: owner/model\n"
+ "task: sft\n"
+ "data:\n"
+ " train: dummy.jsonl\n"
+ "training:\n"
+ " quantization: none\n"
+ " batch_size: 1\n"
+ f" stream_layers: {'true' if stream_layers else 'false'}\n",
+ encoding="utf-8",
+ )
+ (tmp_path / "dummy.jsonl").write_text('{"text": "hi"}\n', encoding="utf-8")
+ seen: list = []
+
+ def recording_detect_device(**kwargs):
+ seen.append(os.environ.get("PYTORCH_CUDA_ALLOC_CONF"))
+ raise _StopHereError
+
+ monkeypatch.setattr("soup_cli.commands.train.detect_device", recording_detect_device)
+ result = CliRunner().invoke(app, ["train", "--config", str(config), "--dry-run"])
+ assert isinstance(result.exception, _StopHereError), (result.output, repr(result.exception))
+ return seen
+
+
+def test_a_streamed_run_on_linux_has_the_hint_before_cuda_starts(monkeypatch, tmp_path):
+ monkeypatch.setattr(sys, "platform", "linux")
+
+ seen = _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True)
+
+ assert len(seen) == 1 and seen[0] is not None and HINT in seen[0]
+
+
+def test_a_run_that_does_not_stream_is_left_alone(monkeypatch, tmp_path):
+ monkeypatch.setattr(sys, "platform", "linux")
+
+ assert _value_at_detect_device(monkeypatch, tmp_path, stream_layers=False) == [None]
+
+
[email protected]("platform", ["win32", "darwin"])
+def test_windows_and_macos_are_left_alone(monkeypatch, tmp_path, platform):
+ monkeypatch.setattr(sys, "platform", platform)
+
+ assert _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True) == [None]
+
+
+def test_a_value_the_user_exported_is_kept(monkeypatch, tmp_path):
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:128")
+
+ seen = _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True)
+
+ assert seen == ["max_split_size_mb:128"]
+
+
+def test_an_empty_export_counts_as_unset(monkeypatch, tmp_path):
+ """``setdefault`` would keep ``""``: the hint stays off and the old reason comes back."""
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "")
+
+ seen = _value_at_detect_device(monkeypatch, tmp_path, stream_layers=True)
+
+ assert seen == [HINT]
+
+
+def test_the_helper_does_not_import_torch(monkeypatch):
+ from soup_cli.utils.gpu import configure_cuda_alloc_hint
+
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setitem(sys.modules, "torch", None) # any `import torch` now raises
+
+ configure_cuda_alloc_hint(stream_layers=True)
+
+ assert os.environ.get("PYTORCH_CUDA_ALLOC_CONF") == HINT
diff --git a/tests/test_issue390_expandable_segments_reason.py b/tests/test_issue390_expandable_segments_reason.py
index 5e99e5ec..b8461200 100644
--- a/tests/test_issue390_expandable_segments_reason.py
+++ b/tests/test_issue390_expandable_segments_reason.py
@@ -96,6 +96,19 @@ class TestExpandableSegmentsStatus:
assert expandable_segments_status() == (True, "")
+ def test_another_exported_value_is_named_as_the_cause(self, fake_cuda, monkeypatch):
+ """#2149: once ``soup train`` sets the hint before CUDA starts, a user's own value
+ is the only way to reach this branch, and "already initialised" is not why."""
+ from soup_cli.utils.layer_stream_runtime import expandable_segments_status
+
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:128")
+ fake_cuda(available=True, initialized=True)
+
+ enabled, why = expandable_segments_status()
+ assert enabled is False
+ assert why == "PYTORCH_CUDA_ALLOC_CONF is set to something else"
+
def test_no_cuda_says_no_cuda(self, fake_cuda, monkeypatch):
from soup_cli.utils.layer_stream_runtime import expandable_segments_status