Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Lambda Cloud controller for ``soup train --cloud lambda`` (#264).
The rendered controller launches an instance with secret-free cloud-init
``user_data``, waits for training over SSH, copies the configured output back
to the caller, and terminates the instance from the caller in a ``finally``
block. The Lambda API key never enters the instance or its boot logs.
"""
from __future__ import annotations
import base64
import os
import re
import sys
import types
from collections.abc import Callable, Mapping
from pathlib import PurePosixPath
from typing import Optional
from soup_cli.cloud._common import (
_MAX_VERSION_LEN,
_VERSION_RE,
CloudPlan,
validate_choice,
write_cloud_stub,
)
from soup_cli.cloud._common import (
validate_path_shape as _validate_path_shape,
)
SUPPORTED_CLOUDS: frozenset[str] = frozenset({"lambda"})
# Lambda instance type names from the public Cloud API catalogue.
_GPU_LAMBDA_NAME: Mapping[str, str] = types.MappingProxyType({
"a10": "gpu_1x_a10",
"a100": "gpu_1x_a100_sxm4",
"h100": "gpu_1x_h100_pcie",
"a6000": "gpu_1x_a6000",
})
SUPPORTED_GPUS: frozenset[str] = frozenset(_GPU_LAMBDA_NAME)
_REMOTE_OUTPUT_RE = re.compile(r"^[A-Za-z0-9._/-]+$")
# Lambda caps user_data at 1 MB. The YAML is base64-encoded inside a small
# shell script, so keep enough headroom for the 4/3 expansion and bootstrap.
_MAX_LAMBDA_CONFIG_BYTES = 700_000
_LAMBDA_SUBMIT_OVERRIDE: Optional[Callable[["CloudPlan"], int]] = None
# The local controller starts in an isolated interpreter (``-I``) with only the
# environment it needs, listed here by name:
# * the Lambda variables the rendered controller reads in ``main()``;
# * the OS variables ``ssh`` / ``scp`` and ``os.path.expanduser`` use to find
# the executables, the home directory (keys, known_hosts) and temp files;
# * ``SSH_AUTH_SOCK``, so a passphrase-protected key still works under
# ``BatchMode=yes``;
# * the proxy and certificate variables ``urllib`` / ``ssl`` read for the
# Lambda API call;
# * the locale variables that pick the output encoding;
# * the dynamic-loader search paths, so an interpreter built against a shared
# ``libpython`` still finds its library.
# ``-I`` also ignores every ``PYTHON*`` variable, so none is forwarded.
_CONTROLLER_ENV_NAMES: frozenset[str] = frozenset({
"LAMBDA_API_KEY",
"LAMBDA_SSH_KEY_NAME",
"LAMBDA_SSH_PRIVATE_KEY",
"LAMBDA_REGION",
"PATH",
"HOME",
"USERPROFILE",
"HOMEDRIVE",
"HOMEPATH",
"USER",
"USERNAME",
"LOGNAME",
"SYSTEMROOT",
"SYSTEMDRIVE",
"WINDIR",
"TEMP",
"TMP",
"TMPDIR",
"SSH_AUTH_SOCK",
"SSL_CERT_FILE",
"SSL_CERT_DIR",
"HTTP_PROXY",
"HTTPS_PROXY",
"ALL_PROXY",
"NO_PROXY",
"http_proxy",
"https_proxy",
"all_proxy",
"no_proxy",
"LANG",
"LC_ALL",
"LC_CTYPE",
"LD_LIBRARY_PATH",
"DYLD_LIBRARY_PATH",
"DYLD_FALLBACK_LIBRARY_PATH",
})
def validate_cloud(name: object) -> str:
"""Validate and normalize the Lambda provider name."""
return validate_choice(name, "cloud", SUPPORTED_CLOUDS)
def validate_gpu(gpu: object) -> str:
"""Validate and normalize a Lambda GPU type."""
return validate_choice(gpu, "gpu", SUPPORTED_GPUS)
def _validate_remote_output(value: object) -> str:
output = _validate_path_shape(value, "output_dir")
path = PurePosixPath(output)
if path.is_absolute() or ".." in path.parts:
raise ValueError("Lambda output_dir must be a relative path without '..'")
if not _REMOTE_OUTPUT_RE.fullmatch(output):
raise ValueError(
"Lambda output_dir may contain only letters, digits, '.', '_', '-', and '/'"
)
return str(path)
def render_lambda_stub(
config_yaml: str,
*,
gpu: str,
output_dir: str,
soup_version: str,
) -> str:
"""Render a local Lambda lifecycle controller with secret-free user data."""
if not isinstance(config_yaml, str):
raise TypeError("config_yaml must be a string")
encoded = config_yaml.encode("utf-8")
if len(encoded) > _MAX_LAMBDA_CONFIG_BYTES:
raise ValueError(
f"config exceeds {_MAX_LAMBDA_CONFIG_BYTES} bytes "
"(too large for Lambda user_data after base64 encoding)"
)
gpu_key = validate_gpu(gpu)
lambda_gpu = _GPU_LAMBDA_NAME[gpu_key]
remote_output = _validate_remote_output(output_dir)
if not isinstance(soup_version, str) or "\x00" in soup_version:
raise ValueError("soup_version must be a NUL-free string")
if len(soup_version) > _MAX_VERSION_LEN or not _VERSION_RE.match(soup_version):
raise ValueError(
f"soup_version must match {_VERSION_RE.pattern} "
f"and be <= {_MAX_VERSION_LEN} chars"
)
cfg_b64 = base64.b64encode(encoded).decode("ascii")
pip_spec = f"soup-cli[train]=={soup_version}"
user_data = (
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"mkdir -p /home/ubuntu/soup\n"
"finish() {\n"
" rc=$?\n"
" printf '%s\\n' \"$rc\" > /home/ubuntu/soup.exit\n"
" chmod -R a+rX /home/ubuntu/soup /home/ubuntu/soup.exit\n"
" trap - EXIT\n"
" exit \"$rc\"\n"
"}\n"
"trap finish EXIT\n"
"apt-get update\n"
"apt-get install -y python3-pip\n"
f"python3 -m pip install {pip_spec!r}\n"
"cd /home/ubuntu/soup\n"
f"printf '%s' {cfg_b64!r} | base64 -d > soup.yaml\n"
"soup train --config soup.yaml --yes 2>&1 | tee /home/ubuntu/soup/train.log\n"
)
lines = [
'"""Generated Lambda Cloud training controller; keep it running until completion."""',
"import base64",
"import json",
"import os",
"import pathlib",
"import re",
"import subprocess",
"import sys",
"import time",
"import urllib.error",
"import urllib.request",
"",
"# Started with -I, which ignores PYTHONIOENCODING and PYTHONUTF8: replace what the",
"# output encoding cannot show, so a log line is printed rather than raising.",
"for _stream in (sys.stdout, sys.stderr):",
" if hasattr(_stream, 'reconfigure'):",
" _stream.reconfigure(errors='replace')",
"",
f"_INSTANCE_TYPE = {lambda_gpu!r}",
f"_USER_DATA = {user_data!r}",
f"_LOCAL_OUTPUT = {output_dir!r}",
f"_REMOTE_OUTPUT = {remote_output!r}",
'_API_BASE = "https://cloud.lambdalabs.com/api/v1"',
'_REGION_RE = re.compile(r"^[a-z0-9-]{1,64}$")',
'_INSTANCE_ID_RE = re.compile(r"^[a-z0-9-]+$")',
"",
"def _request_json(api_key, path, *, method='GET', payload=None):",
" data = None if payload is None else json.dumps(payload).encode('utf-8')",
" request = urllib.request.Request(_API_BASE + path, data=data, method=method)",
" auth = base64.b64encode(f'{api_key}:'.encode('ascii')).decode('ascii')",
" request.add_header('Authorization', f'Basic {auth}')",
" request.add_header('Content-Type', 'application/json')",
" try:",
" with urllib.request.urlopen(request, timeout=60) as response:",
" return json.loads(response.read().decode('utf-8'))",
" except urllib.error.HTTPError as exc:",
" detail = exc.read().decode('utf-8', errors='replace')",
" raise RuntimeError(",
" f'Lambda API {method} {path} failed with HTTP {exc.code}: {detail}'",
" ) from exc",
"",
"def _launch_payload(region, ssh_key_name):",
" return {",
" 'region_name': region,",
" 'instance_type_name': _INSTANCE_TYPE,",
" 'ssh_key_names': [ssh_key_name],",
" 'user_data': _USER_DATA,",
" }",
"",
"def _wait_for_ip(api_key, instance_id):",
" deadline = time.monotonic() + 900",
" while time.monotonic() < deadline:",
" if not _INSTANCE_ID_RE.fullmatch(instance_id):",
" raise ValueError(f'Invalid instance_id format: {instance_id}')",
" record = _request_json(api_key, f'/instances/{instance_id}').get('data', {})",
" ip = record.get('ip')",
" if ip:",
" return str(ip)",
" if record.get('status') in {'terminated', 'unhealthy'}:",
" raise RuntimeError(f'Lambda instance entered {record.get(\"status\")} state')",
" time.sleep(5)",
" raise RuntimeError('Lambda instance did not expose an IP within 15 minutes')",
"",
"def _wait_for_termination(api_key, instance_id):",
" deadline = time.monotonic() + 300",
" while time.monotonic() < deadline:",
" try:",
" if not _INSTANCE_ID_RE.fullmatch(instance_id):",
" raise ValueError(f'Invalid instance_id format: {instance_id}')",
" record = _request_json(",
" api_key, f'/instances/{instance_id}',",
" ).get('data', {})",
" except RuntimeError as exc:",
" if 'HTTP 404' in str(exc):",
" return",
" raise",
" if record.get('status') == 'terminated':",
" return",
" time.sleep(5)",
" raise RuntimeError('Lambda did not confirm termination within 5 minutes')",
"_SSH_OUTAGE_WINDOW = 900",
"_SCP_RETRY_WINDOW = 900",
"_LOG_TAIL_LINES = 50",
"_LOG_TAIL_BYTES = 50_000",
"_LOG_FETCH_TIMEOUT = 15",
"",
"_CONTROL_STRIP_TABLE = {i: None for i in range(0x20) if i not in (0x09, 0x0A, 0x0D)}",
"_CONTROL_STRIP_TABLE[0x7F] = None",
"_CONTROL_STRIP_TABLE.update(dict.fromkeys(range(0x80, 0xA0)))",
"",
"def _strip_control(text):",
" return str(text).translate(_CONTROL_STRIP_TABLE)",
"",
"def _fetch_log_tail(ip, private_key):",
" common = [",
" '-i', private_key, '-o', 'BatchMode=yes',",
" '-o', 'StrictHostKeyChecking=accept-new', '-o', 'ConnectTimeout=15',",
" '-o', 'ServerAliveInterval=30', '-o', 'ServerAliveCountMax=6',",
" ]",
" cmd = f'tail -n {_LOG_TAIL_LINES} /home/ubuntu/soup/train.log'",
" try:",
" res = subprocess.run(",
" ['ssh', *common, f'ubuntu@{ip}', cmd],",
" check=False, capture_output=True, encoding='utf-8', errors='replace',",
" timeout=_LOG_FETCH_TIMEOUT,",
" )",
" if res.returncode != 0:",
" err_lines = res.stderr.strip().splitlines()",
" detail = err_lines[0] if err_lines else f'ssh exited {res.returncode}'",
" clean_detail = _strip_control(detail)[:200]",
" print(f'Could not fetch remote log tail: {clean_detail}')",
" return",
" output = res.stdout",
" if not output:",
" return",
" encoded = output.encode('utf-8')",
" if len(encoded) > _LOG_TAIL_BYTES:",
" output = encoded[-_LOG_TAIL_BYTES:].decode('utf-8', errors='ignore')",
" lines = output.splitlines()[-_LOG_TAIL_LINES:]",
" clean_lines = [_strip_control(line) for line in lines]",
" clean_tail = '\\n'.join(clean_lines).strip()",
" if clean_tail:",
" print(f'Remote log tail (last {_LOG_TAIL_LINES} lines):\\n{clean_tail}')",
" except Exception as exc:",
" clean_exc = _strip_control(str(exc))[:200]",
" print(f'Could not fetch remote log tail: {clean_exc}')",
"",
"def _train_and_copy(ip, private_key):",
" common = [",
" '-i', private_key, '-o', 'BatchMode=yes',",
" # Trust on first use (TOFU) is acceptable for ephemeral instances created here",
" '-o', 'StrictHostKeyChecking=accept-new', '-o', 'ConnectTimeout=15',",
" '-o', 'ServerAliveInterval=30', '-o', 'ServerAliveCountMax=6',",
" ]",
" poll_cmd = (",
' \'case "$(cloud-init status 2>/dev/null)" in \'',
' \'*running*|*"not started"*|*"not run"*) echo RUNNING ;; \'',
' \'*) if [ -s /home/ubuntu/soup.exit ]; then cat /home/ubuntu/soup.exit; \'',
' \'else echo MISSING; fi ;; \'',
' \'esac\'',
" )",
" overall_deadline = time.monotonic() + 90000",
" outage_start = None",
" backoff = 1",
" while True:",
" try:",
" wait = subprocess.run(",
" ['ssh', *common, f'ubuntu@{ip}', poll_cmd],",
" check=False, capture_output=True, text=True, timeout=30,",
" )",
" except subprocess.TimeoutExpired:",
" now = time.monotonic()",
" if outage_start is None:",
" outage_start = now",
" if now - outage_start >= _SSH_OUTAGE_WINDOW:",
" raise RuntimeError(",
" 'lost contact with the instance for 15 minutes: timed out'",
" )",
" time.sleep(min(backoff, 60))",
" backoff = min(backoff * 2, 60)",
" continue",
" if wait.returncode == 255:",
" now = time.monotonic()",
" if outage_start is None:",
" outage_start = now",
" detail = wait.stderr.strip()",
" if now - outage_start >= _SSH_OUTAGE_WINDOW:",
" msg = f': {detail}' if detail else ''",
" raise RuntimeError(",
" f'lost contact with the instance for 15 minutes{msg}'",
" )",
" time.sleep(min(backoff, 60))",
" backoff = min(backoff * 2, 60)",
" continue",
" outage_start = None",
" backoff = 1",
" if wait.returncode != 0:",
" detail = wait.stderr.strip()",
" raise RuntimeError(",
" f'remote check failed with exit {wait.returncode}: {detail}'",
" )",
" status = wait.stdout.strip()",
" if status == 'RUNNING':",
" if time.monotonic() >= overall_deadline:",
" raise RuntimeError('remote training timed out after 25 hours')",
" time.sleep(30)",
" continue",
" try:",
" training_rc = int(status.splitlines()[-1])",
" except (IndexError, ValueError) as exc:",
" raise RuntimeError('remote training did not publish an exit status') from exc",
" if training_rc != 0:",
" _fetch_log_tail(ip, private_key)",
" raise RuntimeError(f'remote soup train exited {training_rc}')",
" break",
" destination = pathlib.Path(_LOCAL_OUTPUT).parent",
" destination.mkdir(parents=True, exist_ok=True)",
" source = f'ubuntu@{ip}:/home/ubuntu/soup/{_REMOTE_OUTPUT}'",
" scp_failure_start = None",
" scp_backoff = 1",
" while True:",
" copied = subprocess.run(",
" ['scp', *common, '-r', source, str(destination)], check=False,",
" )",
" if copied.returncode == 0:",
" break",
" now = time.monotonic()",
" if scp_failure_start is None:",
" scp_failure_start = now",
" if now - scp_failure_start >= _SCP_RETRY_WINDOW:",
" raise RuntimeError(",
" f'checkpoint copy failed after 15 minutes: exit {copied.returncode}'",
" )",
" time.sleep(min(scp_backoff, 60))",
" scp_backoff = min(scp_backoff * 2, 60)",
"",
"def main():",
" api_key = os.environ.get('LAMBDA_API_KEY')",
" ssh_key_name = os.environ.get('LAMBDA_SSH_KEY_NAME')",
" private_key = os.path.expanduser(os.environ.get('LAMBDA_SSH_PRIVATE_KEY', ''))",
" region = os.environ.get('LAMBDA_REGION', 'us-tx-1').lower()",
" if not api_key or not ssh_key_name or not private_key:",
" print('Set LAMBDA_API_KEY, LAMBDA_SSH_KEY_NAME, and LAMBDA_SSH_PRIVATE_KEY.')",
" return 2",
" if not _REGION_RE.fullmatch(region):",
" print('LAMBDA_REGION must contain only lowercase letters, digits, and hyphens.')",
" return 2",
" if not pathlib.Path(private_key).is_file():",
" print('LAMBDA_SSH_PRIVATE_KEY does not name a readable file.')",
" return 2",
" instance_id = None",
" rc = 1",
" try:",
" response = _request_json(",
" api_key, '/instance-operations/launch', method='POST',",
" payload=_launch_payload(region, ssh_key_name),",
" )",
" instance_ids = response.get('data', {}).get('instance_ids', [])",
" if len(instance_ids) != 1:",
" raise RuntimeError(f'Lambda launch returned {instance_ids!r}')",
" instance_id = str(instance_ids[0])",
" print(f'Lambda instance launched: {instance_id}')",
" ip = _wait_for_ip(api_key, instance_id)",
" _train_and_copy(ip, private_key)",
" print(f'Training completed; checkpoints copied to {_LOCAL_OUTPUT}')",
" rc = 0",
" except Exception as exc:",
" print(f'Lambda training failed: {exc}')",
" finally:",
" if instance_id is not None:",
" try:",
" _request_json(",
" api_key, '/instance-operations/terminate', method='POST',",
" payload={'instance_ids': [instance_id]},",
" )",
" _wait_for_termination(api_key, instance_id)",
" print(f'Lambda instance termination verified: {instance_id}')",
" except Exception as exc:",
" print(f'CRITICAL: Lambda termination failed for {instance_id}: {exc}')",
" rc = 1",
" return rc",
"",
"if __name__ == '__main__':",
" sys.exit(main())",
"",
]
return "\n".join(lines)
def plan_lambda_run(
config_path: str,
*,
gpu: str,
output_dir: str,
soup_version: str,
stub_path: str = "soup_lambda_app.py",
) -> CloudPlan:
"""Build a Lambda controller plan from a cwd-contained config."""
from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
enforce_under_cwd_and_no_symlink(config_path, "--config")
with open(config_path, encoding="utf-8") as fh:
config_yaml = fh.read(_MAX_LAMBDA_CONFIG_BYTES + 1)
if len(config_yaml.encode("utf-8")) > _MAX_LAMBDA_CONFIG_BYTES:
raise ValueError(f"config exceeds {_MAX_LAMBDA_CONFIG_BYTES} bytes")
gpu_key = validate_gpu(gpu)
_validate_remote_output(output_dir)
# output_dir is also the LOCAL download root (_LOCAL_OUTPUT in the rendered
# controller). _validate_remote_output already rejects absolute and '..'
# paths by shape, so this adds only the symlink / junction rejection that
# config_path gets above.
enforce_under_cwd_and_no_symlink(output_dir, "output_dir")
_validate_path_shape(stub_path, "stub_path")
stub_text = render_lambda_stub(
config_yaml,
gpu=gpu_key,
output_dir=output_dir,
soup_version=soup_version,
)
return CloudPlan(
cloud="lambda",
gpu=gpu_key,
output_dir=output_dir,
stub_path=stub_path,
stub_text=stub_text,
run_command=f"python -I {stub_path}",
)
def write_stub(plan: CloudPlan) -> str:
"""Write a Lambda controller atomically under the current directory."""
return write_cloud_stub(plan)
_WAITING_NOTICE = (
"Interrupted: waiting for the Lambda controller to terminate the instance...\n"
)
def _notify_waiting_for_controller() -> None:
"""Say that the parent is still waiting, without letting the say-so escape.
This runs from the interrupt handler, so a second Ctrl+C can land on the
write itself; raising from here would abandon the controller mid-cleanup,
which is the whole thing the caller is avoiding.
"""
try:
sys.stderr.write(_WAITING_NOTICE)
sys.stderr.flush()
except BaseException:
pass
def _controller_environment(
environ: Mapping, *, case_insensitive: Optional[bool] = None
) -> dict[str, str]:
"""Return the variables of ``environ`` the local controller is started with.
Only the names in ``_CONTROLLER_ENV_NAMES`` are kept. Windows variable names
are case-insensitive (``SystemRoot`` is ``SYSTEMROOT``), so they are matched
folded to upper case there and exactly everywhere else.
"""
if case_insensitive is None:
case_insensitive = os.name == "nt"
if case_insensitive:
wanted = {name.upper() for name in _CONTROLLER_ENV_NAMES}
return {key: value for key, value in environ.items() if key.upper() in wanted}
return {key: value for key, value in environ.items() if key in _CONTROLLER_ENV_NAMES}
def submit_lambda_run(plan: CloudPlan, *, env: Optional[Mapping] = None) -> int:
"""Run the local lifecycle controller after validating its credentials."""
if not isinstance(plan, CloudPlan):
raise TypeError(f"plan must be a CloudPlan, got {type(plan).__name__}")
if _LAMBDA_SUBMIT_OVERRIDE is not None:
return _LAMBDA_SUBMIT_OVERRIDE(plan)
environ = env if env is not None else os.environ
missing = [
name
for name in (
"LAMBDA_API_KEY",
"LAMBDA_SSH_KEY_NAME",
"LAMBDA_SSH_PRIVATE_KEY",
)
if not environ.get(name)
]
if missing:
raise RuntimeError(
"Lambda Cloud not authenticated/configured. Set "
+ ", ".join(missing)
+ ", then re-run with --cloud-submit."
)
import subprocess
# Not subprocess.run: on KeyboardInterrupt it kills the child 0.25 s later,
# and the child is the controller that terminates the paid instance.
#
# The spawn is INSIDE the try, not before it (#1073). A Ctrl+C reaches the
# parent at whatever bytecode boundary the interpreter happens to be on,
# and the windows a try around proc.wait() alone leaves open -- after
# Popen() returns but before the loop is entered, and on the jump back to
# the top of it -- are ones where the controller is already running. An
# interrupt landing there used to unwind the parent while the instance was
# still being terminated. Once proc exists, every path leads back to wait.
proc = None
while True:
try:
if proc is None:
# Isolated interpreter (-I) and only the environment the
# controller needs; the generated file imports the standard
# library alone, so it does not depend on its own directory
# or on any PYTHON* variable.
proc = subprocess.Popen( # noqa: S603 — argv list, no shell
[sys.executable, "-I", plan.stub_path],
env=_controller_environment(environ),
)
return proc.wait()
except BaseException as exc:
if proc is None:
# The interrupt arrived before the controller existed: there is
# nothing to wait for, so it stays the caller's to handle.
raise
if not isinstance(exc, (KeyboardInterrupt, SystemExit)):
# wait() itself failed. Interrupts are the only thing this loop
# absorbs; retrying a broken wait would spin instead of report.
raise
# The controller received the same Ctrl+C and is terminating the
# instance in its finally block; killing it here would leave the
# instance running. Keep waiting until it exits.
_notify_waiting_for_controller()