Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""The local Lambda controller starts in an isolated interpreter with a minimal environment.
``submit_lambda_run`` starts the generated controller as
``[sys.executable, "-I", <stub>]`` and hands it only the variables it reads
(the Lambda settings), the ones ``ssh`` / ``scp`` need (``PATH``, the home and
temp directories, ``SYSTEMROOT`` on Windows) and the proxy / certificate / agent
variables the network calls honour. Everything else in the parent environment
stays in the parent.
"""
from __future__ import annotations
import os
import subprocess
import sys
from pathlib import Path
import pytest
_LAMBDA_ENV = {
"LAMBDA_API_KEY": "test-key",
"LAMBDA_SSH_KEY_NAME": "test-ssh-key",
"LAMBDA_SSH_PRIVATE_KEY": "/tmp/test-private-key",
}
def _plan(stub_path: str = "x.py"):
from soup_cli.cloud._common import CloudPlan
return CloudPlan(
cloud="lambda",
gpu="a100",
output_dir="./out",
stub_path=stub_path,
stub_text="",
run_command=f"python {stub_path}",
)
class _RecordingPopen:
"""Stands in for ``subprocess.Popen`` and records how the controller is started."""
launches: list[_RecordingPopen] = []
def __init__(self, argv, **kwargs):
self.argv = list(argv)
self.kwargs = kwargs
self.env = kwargs.get("env")
_RecordingPopen.launches.append(self)
def wait(self, timeout=None):
return 0
@pytest.fixture
def launched(monkeypatch):
"""Run ``submit_lambda_run`` against a recording Popen and return the launch."""
_RecordingPopen.launches = []
monkeypatch.setattr(subprocess, "Popen", _RecordingPopen)
def run(env=None):
from soup_cli.cloud.lambda_labs import submit_lambda_run
if env is None:
assert submit_lambda_run(_plan()) == 0
else:
assert submit_lambda_run(_plan(), env=env) == 0
assert len(_RecordingPopen.launches) == 1
return _RecordingPopen.launches[0]
return run
def _set_lambda_env(monkeypatch) -> None:
for name, value in _LAMBDA_ENV.items():
monkeypatch.setenv(name, value)
def test_controller_argv_starts_with_the_isolated_interpreter_flag(launched, monkeypatch):
_set_lambda_env(monkeypatch)
launch = launched()
assert launch.argv[0] == sys.executable
assert launch.argv[1] == "-I"
assert launch.argv == [sys.executable, "-I", "x.py"]
def test_printed_manual_run_command_uses_the_isolated_interpreter_flag(tmp_path, monkeypatch):
from soup_cli.cloud.lambda_labs import plan_lambda_run
monkeypatch.chdir(tmp_path)
(tmp_path / "soup.yaml").write_text("base: x\n", encoding="utf-8")
plan = plan_lambda_run(
"soup.yaml", gpu="a100", output_dir="./out", soup_version="0.76.0", stub_path="ctl.py"
)
# The command a user is told to run starts the controller the way
# ``--cloud-submit`` does: ``python -I <stub>``.
assert plan.run_command == "python -I ctl.py"
def test_controller_gets_the_lambda_and_os_variables_it_reads(launched, monkeypatch):
_set_lambda_env(monkeypatch)
monkeypatch.setenv("LAMBDA_REGION", "us-west-1")
monkeypatch.setenv("PATH", os.environ["PATH"])
monkeypatch.setenv("HOME", "/home/someone")
monkeypatch.setenv("USERPROFILE", "C:\\Users\\someone")
monkeypatch.setenv("SYSTEMROOT", "C:\\Windows")
monkeypatch.setenv("TEMP", "/tmp/t")
monkeypatch.setenv("TMP", "/tmp/t")
env = launched().env
assert env["LAMBDA_API_KEY"] == "test-key"
assert env["LAMBDA_SSH_KEY_NAME"] == "test-ssh-key"
assert env["LAMBDA_SSH_PRIVATE_KEY"] == "/tmp/test-private-key"
assert env["LAMBDA_REGION"] == "us-west-1"
assert env["PATH"] == os.environ["PATH"]
for name in ("HOME", "USERPROFILE", "SYSTEMROOT", "TEMP", "TMP"):
assert name in env, f"{name} must reach ssh / scp and the interpreter"
def test_controller_env_leaves_an_unrelated_parent_variable_behind(launched, monkeypatch):
_set_lambda_env(monkeypatch)
monkeypatch.setenv("SOUP_TEST_UNRELATED", "1")
env = launched().env
assert "SOUP_TEST_UNRELATED" not in env
assert env["LAMBDA_API_KEY"] == "test-key"
def test_controller_env_is_built_from_a_supplied_mapping_too(launched):
supplied = dict(_LAMBDA_ENV, SOUP_TEST_UNRELATED="1", PATH=os.environ["PATH"])
env = launched(env=supplied).env
assert "SOUP_TEST_UNRELATED" not in env
assert env["PATH"] == os.environ["PATH"]
assert env["LAMBDA_API_KEY"] == "test-key"
def test_controller_env_is_a_new_mapping_not_the_parent_environment(launched, monkeypatch):
_set_lambda_env(monkeypatch)
env = launched().env
assert env is not os.environ
assert isinstance(env, dict)
assert set(env) <= set(os.environ)
# Whatever else the parent has, the controller sees no more than the allowlist.
from soup_cli.cloud.lambda_labs import _controller_environment
assert set(env) == set(_controller_environment(os.environ))
@pytest.mark.parametrize(
"name",
["HF_TOKEN", "WANDB_API_KEY", "MODAL_TOKEN_SECRET", "PYTHONPATH", "PYTHONSTARTUP"],
)
def test_other_tool_settings_in_the_parent_are_not_forwarded(launched, monkeypatch, name):
_set_lambda_env(monkeypatch)
monkeypatch.setenv(name, "value")
assert name not in launched().env
@pytest.mark.parametrize(
"name",
[
"SSH_AUTH_SOCK",
"SSL_CERT_FILE",
"SSL_CERT_DIR",
"HTTPS_PROXY",
"https_proxy",
"NO_PROXY",
"no_proxy",
"LANG",
"LD_LIBRARY_PATH",
"DYLD_LIBRARY_PATH",
"DYLD_FALLBACK_LIBRARY_PATH",
],
)
def test_network_agent_locale_and_library_settings_are_forwarded(launched, monkeypatch, name):
_set_lambda_env(monkeypatch)
monkeypatch.setenv(name, "value")
env = launched().env
# os.environ upper-cases names on Windows, so ``https_proxy`` arrives there
# as ``HTTPS_PROXY``; everywhere else the name is kept as it was set.
assert env[name.upper() if os.name == "nt" else name] == "value"
class TestControllerEnvironmentNameMatching:
def test_windows_names_are_matched_without_regard_to_case(self):
from soup_cli.cloud.lambda_labs import _controller_environment
env = _controller_environment(
{"SystemRoot": "C:\\Windows", "Path": "C:\\bin", "Other": "1"},
case_insensitive=True,
)
assert env == {"SystemRoot": "C:\\Windows", "Path": "C:\\bin"}
def test_other_platforms_match_names_exactly(self):
from soup_cli.cloud.lambda_labs import _controller_environment
env = _controller_environment(
{"SystemRoot": "x", "Path": "y", "PATH": "z", "HOME": "h"},
case_insensitive=False,
)
assert env == {"PATH": "z", "HOME": "h"}
def test_default_follows_the_platform(self):
from soup_cli.cloud.lambda_labs import _controller_environment
env = _controller_environment({"path": "p", "PATH": "P"})
if os.name == "nt":
assert env == {"path": "p", "PATH": "P"}
else:
assert env == {"PATH": "P"}
# The generated controller must start under ``-I`` with the minimal environment:
# it imports the standard library only and reads nothing it was not given. These
# runs stop at the controller's own argument checks, so no network call is made.
def _render_controller(tmp_path: Path) -> Path:
from soup_cli.cloud.lambda_labs import render_lambda_stub
stub = tmp_path / "soup_lambda_app.py"
stub.write_text(
render_lambda_stub(
"base: x\n", gpu="a100", output_dir="./out", soup_version="0.76.0"
),
encoding="utf-8",
)
return stub
def _parent_env(**extra: str) -> dict[str, str]:
"""The test process environment without any Lambda setting, plus ``extra``.
A developer machine may have the real ``LAMBDA_*`` variables set; the
controller runs below must never reach the network because of them.
"""
env = {
key: value
for key, value in os.environ.items()
if not key.upper().startswith("LAMBDA_") and key.upper() != "PYTHONSAFEPATH"
}
env.update(extra)
return env
def _run_controller(stub: Path, parent_env: dict[str, str]) -> subprocess.CompletedProcess:
from soup_cli.cloud.lambda_labs import _controller_environment
return subprocess.run(
[sys.executable, "-I", str(stub)],
env=_controller_environment(parent_env),
cwd=stub.parent,
capture_output=True,
text=True,
timeout=120,
)
def test_rendered_controller_starts_under_isolated_mode(tmp_path):
stub = _render_controller(tmp_path)
result = _run_controller(stub, _parent_env())
assert result.returncode == 2, (result.stdout, result.stderr)
assert "Set LAMBDA_API_KEY, LAMBDA_SSH_KEY_NAME, and LAMBDA_SSH_PRIVATE_KEY." in result.stdout
def test_rendered_controller_reads_its_settings_from_the_minimal_environment(tmp_path):
stub = _render_controller(tmp_path)
parent_env = _parent_env(
SOUP_TEST_UNRELATED="1",
LAMBDA_API_KEY="k",
LAMBDA_SSH_KEY_NAME="n",
LAMBDA_SSH_PRIVATE_KEY=str(tmp_path / "missing-key"),
)
result = _run_controller(stub, parent_env)
assert result.returncode == 2, (result.stdout, result.stderr)
assert "LAMBDA_SSH_PRIVATE_KEY does not name a readable file." in result.stdout
def test_rendered_controller_validates_the_region_it_is_given(tmp_path):
stub = _render_controller(tmp_path)
parent_env = _parent_env(
LAMBDA_API_KEY="k",
LAMBDA_SSH_KEY_NAME="n",
LAMBDA_SSH_PRIVATE_KEY=str(tmp_path / "missing-key"),
LAMBDA_REGION="Bad Region",
)
result = _run_controller(stub, parent_env)
assert result.returncode == 2, (result.stdout, result.stderr)
assert (
"LAMBDA_REGION must contain only lowercase letters, digits, and hyphens."
in result.stdout
)
def test_rendered_controller_does_not_import_modules_from_its_own_directory(tmp_path):
stub = _render_controller(tmp_path)
# A module that shares a standard-library name and sits next to the stub.
# Under -I the directory of the script is not on sys.path, so it is never
# imported; without -I it would be the ``json`` the controller picks up.
(tmp_path / "json.py").write_text(
"raise RuntimeError('imported from the controller directory')\n",
encoding="utf-8",
)
isolated = _run_controller(stub, _parent_env())
plain = subprocess.run(
[sys.executable, str(stub)],
env=_parent_env(),
cwd=tmp_path,
capture_output=True,
text=True,
timeout=120,
)
assert isolated.returncode == 2, (isolated.stdout, isolated.stderr)
assert "imported from the controller directory" not in isolated.stderr
# The control run proves the sibling module is really on the path without -I.
assert "imported from the controller directory" in plain.stderr
# ``-I`` also ignores PYTHONUTF8 and PYTHONIOENCODING, so the controller's output goes
# through the platform default encoding (the ANSI code page on a Windows pipe). The
# rendered file reconfigures stdout and stderr to replace what that encoding cannot
# show, so a log line with such a character is printed instead of raising.
#
# The harness runs the real controller code under ``-I``: ``narrow`` first puts both
# streams on an ASCII pipe encoding that raises on anything else (the same failure a
# Windows code page gives, on every platform); ``native`` keeps whatever the platform
# default is. ``stdout`` then drives ``main()``'s own print path with a failing Lambda
# call; ``stderr`` writes one line after the controller has been loaded.
_ENCODING_HARNESS = r'''
import sys
stream, mode, stub = sys.argv[1:4]
if mode == "narrow":
for handle in (sys.stdout, sys.stderr):
handle.reconfigure(encoding="ascii", errors="strict")
namespace = {"__name__": "controller_under_test"}
with open(stub, encoding="utf-8") as fh:
source = fh.read()
exec(compile(source, stub, "exec"), namespace)
text = "remote said: \u2588 \u00e9 \u65e5"
if stream == "stdout":
def _fail(*args, **kwargs):
raise RuntimeError(text)
namespace["_request_json"] = _fail
sys.exit(namespace["main"]())
sys.stderr.write("stderr line: " + text + "\n")
sys.stderr.flush()
'''
def _run_encoding_harness(tmp_path: Path, stream: str, mode: str):
from soup_cli.cloud.lambda_labs import _controller_environment
stub = _render_controller(tmp_path)
harness = tmp_path / "encoding_harness.py"
harness.write_text(_ENCODING_HARNESS, encoding="utf-8")
key = tmp_path / "key"
key.write_text("not a real key\n", encoding="utf-8")
parent_env = _parent_env(
LAMBDA_API_KEY="k",
LAMBDA_SSH_KEY_NAME="n",
LAMBDA_SSH_PRIVATE_KEY=str(key),
)
# No PYTHONIOENCODING / PYTHONUTF8: the controller environment never carries them.
return subprocess.run(
[sys.executable, "-I", str(harness), stream, mode, str(stub)],
env=_controller_environment(parent_env),
cwd=tmp_path,
capture_output=True,
timeout=120,
)
def test_stdout_prints_a_line_its_encoding_cannot_show(tmp_path):
result = _run_encoding_harness(tmp_path, "stdout", "narrow")
stdout = result.stdout.decode("ascii")
stderr = result.stderr.decode("utf-8", errors="replace")
assert "UnicodeEncodeError" not in stderr, stderr
assert result.returncode == 1, (stdout, stderr)
assert "Lambda training failed: remote said: ? ? ?" in stdout
def test_stderr_writes_a_line_its_encoding_cannot_show(tmp_path):
result = _run_encoding_harness(tmp_path, "stderr", "narrow")
stderr = result.stderr.decode("ascii")
assert "UnicodeEncodeError" not in stderr, stderr
assert result.returncode == 0, (result.stdout, stderr)
assert "stderr line: remote said: ? ? ?" in stderr
@pytest.mark.parametrize("stream", ["stdout", "stderr"])
def test_output_with_the_platform_default_pipe_encoding_does_not_raise(tmp_path, stream):
# Windows pipes use the ANSI code page, which cannot show U+2588 or U+65E5; on
# other platforms the default is usually UTF-8 and this is a smoke run.
result = _run_encoding_harness(tmp_path, stream, "native")
stdout = result.stdout.decode("utf-8", errors="replace")
stderr = result.stderr.decode("utf-8", errors="replace")
assert "UnicodeEncodeError" not in stderr, stderr
if stream == "stdout":
assert result.returncode == 1, (stdout, stderr)
assert "Lambda training failed: remote said:" in stdout
else:
assert result.returncode == 0, (stdout, stderr)
assert "stderr line: remote said:" in stderr