Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""v0.43.0 Part A — Tracker integrations + PostHog telemetry opt-out.
Closed allowlist of HF Trainer `report_to` backends Soup recognises.
Adds mlflow / swanlab / trackio to the legacy `wandb` / `tensorboard` / `none`
set. Live integrations rely on HF Trainer's built-in callbacks (mlflow,
swanlab) plus the third-party `trackio` callback when installed; Soup only
validates the name and surfaces a friendly error when the backing package
is missing.
Telemetry: opt-out via `SOUP_TELEMETRY=0` env var. Default is OFF until a
public privacy policy ships — `is_telemetry_enabled` returns False unless
the user explicitly enables it. Hardware-info-only payload schema lives in
`build_telemetry_payload` for documentation/testing; no network calls in
v0.43.0 (PostHog wire-up deferred to v0.43.1).
"""
from __future__ import annotations
import math
import os
import platform
from types import MappingProxyType
from typing import Mapping
# Closed allowlist of report_to backends.
_REPORT_TO_BACKENDS: Mapping[str, str | None] = MappingProxyType({
"none": None,
"wandb": "wandb",
"tensorboard": "tensorboard",
"mlflow": "mlflow",
"swanlab": "swanlab",
"trackio": "trackio",
})
SUPPORTED_TRACKERS = frozenset(_REPORT_TO_BACKENDS.keys())
# v0.43.0 additions (HF-native wandb/tensorboard already supported).
NEW_TRACKERS_V0_43 = frozenset({"mlflow", "swanlab", "trackio"})
_MAX_NAME_LEN = 32
def validate_tracker_name(name: object) -> str:
"""Validate and lowercase a `report_to` tracker name.
Returns the canonical lower-cased name. Raises ValueError on invalid
input. Mirrors v0.41.0 `validate_optimizer_name` policy.
"""
if not isinstance(name, str):
raise ValueError(f"tracker name must be a string, got {type(name).__name__}")
if not name:
raise ValueError("tracker name must not be empty")
if "\x00" in name:
raise ValueError("tracker name must not contain null bytes")
if len(name) > _MAX_NAME_LEN:
raise ValueError(
f"tracker name length {len(name)} exceeds max {_MAX_NAME_LEN}"
)
canonical = name.lower()
if canonical not in SUPPORTED_TRACKERS:
supported = ", ".join(sorted(SUPPORTED_TRACKERS))
raise ValueError(
f"unknown tracker '{name}'. Supported: {supported}"
)
return canonical
def required_tracker_package(name: str) -> str | None:
"""Return the pip-installable package name for a tracker, or None.
Non-string input returns None (mirrors `is_new_v0_43_tracker`).
"""
if not isinstance(name, str):
return None
return _REPORT_TO_BACKENDS.get(name.lower())
def is_new_v0_43_tracker(name: object) -> bool:
"""True if the name is an additive v0.43.0 tracker, False otherwise."""
if not isinstance(name, str):
return False
return name.lower() in NEW_TRACKERS_V0_43
# --- Telemetry (opt-out, default OFF) ----------------------------------
_TELEMETRY_ENV_VAR = "SOUP_TELEMETRY"
def is_telemetry_enabled(env: Mapping[str, str] | None = None) -> bool:
"""Telemetry is opt-IN until v0.43.1 ships the network code.
The roadmap entry calls this opt-out, but until the privacy policy
+ PostHog wire-up land we keep it default-OFF so no payload is built
or sent. Users may enable explicitly with `SOUP_TELEMETRY=1`.
"""
source = env if env is not None else os.environ
raw = source.get(_TELEMETRY_ENV_VAR)
if raw is None:
return False
val = raw.strip().lower()
if val in {"1", "true", "yes", "on"}:
return True
return False
def get_or_create_distinct_id() -> str:
"""Return the anonymous telemetry UUID, generating and persisting it if needed.
The identifier is stored at ``~/.soup/telemetry_id``. It contains NO
user-identifying or hardware-identifying information — it is a random
UUID4 generated locally for event deduplication.
All filesystem operations fail soft so telemetry can NEVER crash training.
"""
import uuid # noqa: PLC0415
from pathlib import Path # noqa: PLC0415
from soup_cli.utils.constants import SOUP_DIR # noqa: PLC0415
id_file = Path.home() / SOUP_DIR / "telemetry_id"
try:
if id_file.exists():
saved = id_file.read_text(encoding="utf-8").strip()
# Validate format strictly: must be a valid UUID
if str(uuid.UUID(saved)) == saved.lower():
return saved
except Exception:
pass
new_id = str(uuid.uuid4())
try:
id_file.parent.mkdir(parents=True, exist_ok=True)
id_file.write_text(new_id + "\n", encoding="utf-8")
except Exception:
pass
return new_id
def build_telemetry_payload(
*,
soup_version: str,
command: str,
duration_seconds: float | int | None = None,
distinct_id: str | None = None,
) -> dict:
"""Build the hardware-info-only telemetry payload.
The payload contains NO user data, dataset paths, model names, or
config contents. Documented schema:
- `soup_version`: caller-supplied
- `command`: top-level CLI command (e.g. `train`, `data ingest`)
- `python`: major.minor only
- `os`: platform.system()
- `arch`: platform.machine()
- `duration_seconds`: optional, finite float / int / None
- `distinct_id`: anonymous persistent UUID (or ephemeral in-memory
UUID when telemetry is disabled, avoiding filesystem side effects)
Raises ValueError for non-string `command` / `soup_version` and for
non-finite `duration_seconds`.
"""
if not isinstance(soup_version, str) or not soup_version:
raise ValueError("soup_version must be a non-empty string")
if "\x00" in soup_version:
raise ValueError("soup_version must not contain null bytes")
if not isinstance(command, str) or not command:
raise ValueError("command must be a non-empty string")
if "\x00" in command:
raise ValueError("command must not contain null bytes")
if duration_seconds is not None:
# bool is a subclass of int — reject explicitly (project policy)
if isinstance(duration_seconds, bool) or not isinstance(
duration_seconds, (int, float)
):
raise ValueError("duration_seconds must be int / float / None")
if not math.isfinite(float(duration_seconds)):
raise ValueError("duration_seconds must be finite")
if duration_seconds < 0:
raise ValueError("duration_seconds must be >= 0")
if distinct_id is not None:
resolved_distinct_id = str(distinct_id)
elif is_telemetry_enabled():
resolved_distinct_id = get_or_create_distinct_id()
else:
import uuid # noqa: PLC0415
resolved_distinct_id = str(uuid.uuid4())
py = platform.python_version_tuple()
py_major_minor = f"{py[0]}.{py[1]}"
return {
"soup_version": soup_version,
"command": command,
"python": py_major_minor,
"os": platform.system(),
"arch": platform.machine(),
"duration_seconds": (
float(duration_seconds) if duration_seconds is not None else None
),
"distinct_id": resolved_distinct_id,
}
# v0.53.8 #90 — PostHog telemetry live wiring.
# Opt-IN via SOUP_TELEMETRY=1; silent-fail on any network/transport error
# so telemetry can NEVER crash training. 1s hard timeout, HTTPS-only.
_POSTHOG_HOST = "https://us.i.posthog.com"
_POSTHOG_ENDPOINT = f"{_POSTHOG_HOST}/i/v0/e/"
# v0.53.10 #154 — bundled public write-only project key for Soup CLI
# telemetry. The key is INTENTIONALLY hard-coded: PostHog "phc_*" keys are
# write-only (cannot read events back); rotating it requires a release.
# Operators wanting to point telemetry at their own PostHog project should
# set ``SOUP_POSTHOG_KEY`` AND ``SOUP_POSTHOG_ENDPOINT`` together; both env
# vars are validated by :func:`_resolve_posthog_target`.
_POSTHOG_DEFAULT_KEY = "phc_soup_public_write_only"
_TELEMETRY_TIMEOUT_S = 1.0
def is_placeholder_posthog_key(key: object) -> bool:
"""Return True if ``key`` is the unprovisioned placeholder key."""
if not isinstance(key, str) or not key:
return True
clean = key.strip()
return clean == _POSTHOG_DEFAULT_KEY or clean.startswith("phc_soup_public_write_only")
# Sentinel for "caller did not pass an endpoint, fall back to default + env".
_POSTHOG_ENDPOINT_DEFAULT = object()
def _resolve_posthog_target(
api_key: str | None,
endpoint: object = _POSTHOG_ENDPOINT_DEFAULT,
env: dict[str, str] | None = None,
) -> tuple[str, str] | None:
"""Resolve ``(key, endpoint)`` from explicit args + ``SOUP_POSTHOG_*`` env.
v0.53.10 #154 — adds env-var overrides for the bundled defaults so users
on private PostHog instances can point Soup telemetry at their own
project without a code change. Precedence:
1. Explicit ``api_key`` / ``endpoint`` kwargs (caller wins).
2. ``SOUP_POSTHOG_KEY`` env var (overrides ``_POSTHOG_DEFAULT_KEY``).
3. ``SOUP_POSTHOG_ENDPOINT`` env var (overrides
``_POSTHOG_ENDPOINT``; must be HTTPS + pass the v0.51.0 SSRF policy).
4. Bundled defaults.
Returns ``None`` when any input fails validation (silent no-op so
telemetry can never crash training).
"""
import os # noqa: PLC0415 — local lazy import
src = env if env is not None else os.environ
# Endpoint resolution: explicit caller > env override > default.
# Use a sentinel default so a caller who passes
# ``endpoint=_POSTHOG_ENDPOINT`` (locking in the default) is NOT silently
# overridden by ``SOUP_POSTHOG_ENDPOINT`` (code-review HIGH fix).
if endpoint is _POSTHOG_ENDPOINT_DEFAULT:
env_endpoint = src.get("SOUP_POSTHOG_ENDPOINT")
resolved_endpoint = env_endpoint or _POSTHOG_ENDPOINT
else:
resolved_endpoint = endpoint
if not isinstance(resolved_endpoint, str):
return None
if not _telemetry_endpoint_is_safe(resolved_endpoint):
return None
# Key resolution: explicit caller > env override > default.
if api_key is not None:
key = api_key
else:
key = src.get("SOUP_POSTHOG_KEY") or _POSTHOG_DEFAULT_KEY
if not isinstance(key, str) or not key:
return None
# Reject control chars / whitespace in the key — defends against an
# operator dropping ``\nAuthorization:...`` into SOUP_POSTHOG_KEY.
if "\x00" in key or any(ord(c) < 0x20 for c in key) or len(key) > 256:
return None
return key, resolved_endpoint
# Internal / non-routable TLD suffixes rejected without DNS resolution (#599).
_INTERNAL_TLD_SUFFIXES: tuple[str, ...] = (
".local",
".internal",
".localhost",
".lan",
".home",
".corp",
".intranet",
)
_INTERNAL_TLD_EXACT: frozenset[str] = frozenset({
"local",
"internal",
"localhost",
"lan",
"home",
"corp",
"intranet",
})
def _is_trusted_posthog_domain(host: str) -> bool:
"""Return True if ``host`` belongs to the trusted PostHog domain hierarchy."""
return host == "posthog.com" or host.endswith(".posthog.com")
def _resolve_host_ips(host: str, *, timeout: float = 1.0) -> list[str] | None:
"""Resolve ``host`` to IP addresses within ``timeout`` seconds.
Returns a list of IP strings, or ``None`` on resolution failure,
resolver error, or timeout (fail-closed).
"""
import socket # noqa: PLC0415
import threading # noqa: PLC0415
result: list[str] = []
error: list[Exception] = []
def _worker() -> None:
try:
info = socket.getaddrinfo(host, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
for item in info:
sockaddr = item[4]
ip = sockaddr[0]
if ip:
result.append(ip)
except Exception as exc:
error.append(exc)
thread = threading.Thread(target=_worker, daemon=True)
thread.start()
thread.join(timeout=timeout)
if thread.is_alive() or error or not result:
return None
return result
def _telemetry_endpoint_is_safe(endpoint: str) -> bool:
"""Tiered endpoint guard for telemetry destinations (#593, #599).
Three-tier validation architecture:
1. **Primary Control (Trusted Allowlist):** Canonical PostHog domains
(``us.i.posthog.com``, ``eu.i.posthog.com``, ``*.posthog.com``) are
accepted immediately. This guarantees sub-millisecond validation with
zero network requests and zero risk of resolver timeout on default paths.
2. **Static Syntactic & Suffix Guards:** For custom endpoint URLs, reject
loopback names (``localhost``), literal private/link-local/loopback IPs
(including abbreviated/hex/octal IPv4 forms), and internal TLD suffixes
(``.local``, ``.internal``, ``.lan``, ``.home``, ``.corp``, etc.) without
performing network lookups.
3. **Defence-in-Depth DNS Resolution:** For custom non-default FQDNs, resolve
the host to IP addresses with a bounded timeout and fail closed. Rejects if
any resolved address is a loopback, private, or link-local IP. This raises
the bar against hostname indirection (e.g. ``10.0.0.1.nip.io``,
``localtest.me``). Note: this does not prevent DNS rebinding across separate
lookups, but provides defence in depth for custom endpoint configurations.
The HTTPS-only requirement (``startswith("https://")``) is applied first.
"""
if not isinstance(endpoint, str) or not endpoint.startswith("https://"):
return False
# Layer 1: baseline sanitization (CRLF, null bytes, types, 0.0.0.0).
try:
from soup_cli.utils.hubs import validate_hub_endpoint
validate_hub_endpoint(endpoint, hub="telemetry")
except (TypeError, ValueError):
return False
from urllib.parse import urlparse # noqa: PLC0415
from soup_cli.utils.hubs import ( # noqa: PLC0415
_LOOPBACK_HOSTS,
_is_private_or_link_local,
)
host = (urlparse(endpoint).hostname or "").lower().rstrip(".")
if not host:
return False
# Tier 1: Primary Control — Trusted PostHog domain allowlist.
# Eliminates DNS lookups for default configurations (99%+ of runs).
if _is_trusted_posthog_domain(host):
return True
# Tier 2: Static rejection (literal IPs, loopback hosts, internal TLDs).
if host in _LOOPBACK_HOSTS:
return False
if _is_private_or_link_local(host):
return False
if host in _INTERNAL_TLD_EXACT or host.endswith(_INTERNAL_TLD_SUFFIXES):
return False
# Tier 3: Defence-in-depth DNS resolution for custom non-default FQDNs.
# Raises the bar against hostname indirection; fails closed on error/timeout.
resolved_ips = _resolve_host_ips(host)
if not resolved_ips:
return False
for ip in resolved_ips:
clean_ip = ip.rstrip(".")
if clean_ip in _LOOPBACK_HOSTS or _is_private_or_link_local(clean_ip):
return False
return True
def send_telemetry_payload(
payload: dict[str, object],
*,
api_key: str | None = None,
timeout: float = _TELEMETRY_TIMEOUT_S,
endpoint: object = _POSTHOG_ENDPOINT_DEFAULT,
) -> bool:
"""POST ``payload`` to PostHog if telemetry is enabled, else no-op.
Returns ``True`` on a 2xx response, ``False`` on any failure or skip.
NEVER raises — telemetry is best-effort and must never crash training.
Args:
payload: dict built by :func:`build_telemetry_payload`. Required keys
are validated upstream by the builder.
api_key: PostHog project key. Defaults to the bundled write-only key.
timeout: socket connect and read timeout in seconds (default 1.0 s;
DNS resolution excluded).
endpoint: full PostHog capture URL (must be HTTPS).
"""
if not is_telemetry_enabled():
return False
if not isinstance(payload, dict) or not payload:
return False
if isinstance(timeout, bool) or not isinstance(timeout, (int, float)):
return False
if not math.isfinite(float(timeout)) or timeout <= 0:
return False
# v0.53.10 #154 — resolve key + endpoint via env-override-aware helper.
# Returns ``None`` when either input fails validation; treat as silent
# no-op so telemetry remains best-effort.
resolved = _resolve_posthog_target(api_key, endpoint)
if resolved is None:
return False
key, endpoint = resolved
if is_placeholder_posthog_key(key):
import sys # noqa: PLC0415
try:
sys.stderr.write("Notice: telemetry is not yet live.\n")
except Exception:
pass
return False
body = {
"api_key": key,
"event": payload.get("command", "soup_event"),
"properties": {k: v for k, v in payload.items() if k != "command"},
}
try:
import json # noqa: PLC0415
from urllib import request # noqa: PLC0415
encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode(
"utf-8"
)
req = request.Request(
endpoint,
data=encoded,
headers={"Content-Type": "application/json"},
method="POST",
)
with request.urlopen(req, timeout=float(timeout)) as response:
status = getattr(response, "status", None)
if not isinstance(status, int):
status = response.getcode()
return isinstance(status, int) and 200 <= status < 300
except Exception: # noqa: BLE001 — telemetry must never crash training
return False
# v0.53.8 #89 — Friendly missing-dep panel for HF Trainer `--tracker`.
# When user passes `--tracker mlflow` without mlflow installed, HF raises
# a generic ImportError mid-training; this helper lets the CLI surface a
# pip-install advisory BEFORE construction.
def tracker_missing_dep_message(name: str) -> str | None:
"""Return a friendly install advisory for ``name`` if the package is
missing, else None.
Always returns ``None`` for `wandb` / `tensorboard` / `none` (the
legacy backends), since those are part of the standard HF Trainer
extra and not part of v0.43.0's additive set.
"""
if not isinstance(name, str):
return None
canonical = name.lower()
if canonical not in NEW_TRACKERS_V0_43:
return None
pkg = required_tracker_package(canonical)
if not pkg:
return None
# Use ``importlib.util.find_spec`` (non-executing probe) so we don't
# incur side effects from the tracker's top-level module (e.g. swanlab
# initialises network threads on import). ``sys.modules[pkg] = None``
# raises ``ValueError`` on find_spec — treat that as missing too so
# tests can simulate the absent-package path without subprocess.
import importlib.util
import sys
sentinel = object()
cached = sys.modules.get(pkg, sentinel)
if cached is None:
missing = True
elif cached is not sentinel:
# Module is already imported (or test injected a real-shaped mock).
missing = False
else:
try:
missing = importlib.util.find_spec(pkg) is None
except (ImportError, ValueError):
missing = True
if missing:
return (
f"--tracker {canonical} requires the '{pkg}' package. "
f"Install with: pip install soup-cli[trackers] "
f"(or pip install {pkg})"
)
return None
def resolve_report_to(
*,
wandb: bool = False,
tensorboard: bool = False,
tracker: str | None = None,
) -> str:
"""Resolve the HF Trainer `report_to` value from CLI flags + --tracker.
Mutual-exclusion: only one of (wandb, tensorboard, tracker) may be set.
Empty string / None on `tracker` is treated as unset.
"""
set_count = sum(
1
for x in (
bool(wandb),
bool(tensorboard),
bool(tracker) if isinstance(tracker, str) and tracker else False,
)
if x
)
if set_count > 1:
raise ValueError(
"--wandb, --tensorboard, and --tracker are mutually exclusive"
)
if wandb:
return "wandb"
if tensorboard:
return "tensorboard"
if tracker:
return validate_tracker_name(tracker)
return "none"