Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

20.0 KB
Raw Blame
"""v0.43.0 Part A — Tracker integrations + PostHog telemetry opt-out.

Closed allowlist of HF Trainer `report_to` backends Soup recognises.
Adds mlflow / swanlab / trackio to the legacy `wandb` / `tensorboard` / `none`
set. Live integrations rely on HF Trainer's built-in callbacks (mlflow,
swanlab) plus the third-party `trackio` callback when installed; Soup only
validates the name and surfaces a friendly error when the backing package
is missing.

Telemetry: opt-out via `SOUP_TELEMETRY=0` env var. Default is OFF until a
public privacy policy ships — `is_telemetry_enabled` returns False unless
the user explicitly enables it. Hardware-info-only payload schema lives in
`build_telemetry_payload` for documentation/testing; no network calls in
v0.43.0 (PostHog wire-up deferred to v0.43.1).
"""
from __future__ import annotations

import math
import os
import platform
from types import MappingProxyType
from typing import Mapping

# Closed allowlist of report_to backends.
_REPORT_TO_BACKENDS: Mapping[str, str | None] = MappingProxyType({
    "none": None,
    "wandb": "wandb",
    "tensorboard": "tensorboard",
    "mlflow": "mlflow",
    "swanlab": "swanlab",
    "trackio": "trackio",
})

SUPPORTED_TRACKERS = frozenset(_REPORT_TO_BACKENDS.keys())

# v0.43.0 additions (HF-native wandb/tensorboard already supported).
NEW_TRACKERS_V0_43 = frozenset({"mlflow", "swanlab", "trackio"})

_MAX_NAME_LEN = 32


def validate_tracker_name(name: object) -> str:
    """Validate and lowercase a `report_to` tracker name.

    Returns the canonical lower-cased name. Raises ValueError on invalid
    input. Mirrors v0.41.0 `validate_optimizer_name` policy.
    """
    if not isinstance(name, str):
        raise ValueError(f"tracker name must be a string, got {type(name).__name__}")
    if not name:
        raise ValueError("tracker name must not be empty")
    if "\x00" in name:
        raise ValueError("tracker name must not contain null bytes")
    if len(name) > _MAX_NAME_LEN:
        raise ValueError(
            f"tracker name length {len(name)} exceeds max {_MAX_NAME_LEN}"
        )
    canonical = name.lower()
    if canonical not in SUPPORTED_TRACKERS:
        supported = ", ".join(sorted(SUPPORTED_TRACKERS))
        raise ValueError(
            f"unknown tracker '{name}'. Supported: {supported}"
        )
    return canonical


def required_tracker_package(name: str) -> str | None:
    """Return the pip-installable package name for a tracker, or None.

    Non-string input returns None (mirrors `is_new_v0_43_tracker`).
    """
    if not isinstance(name, str):
        return None
    return _REPORT_TO_BACKENDS.get(name.lower())


def is_new_v0_43_tracker(name: object) -> bool:
    """True if the name is an additive v0.43.0 tracker, False otherwise."""
    if not isinstance(name, str):
        return False
    return name.lower() in NEW_TRACKERS_V0_43


# --- Telemetry (opt-out, default OFF) ----------------------------------

_TELEMETRY_ENV_VAR = "SOUP_TELEMETRY"


def is_telemetry_enabled(env: Mapping[str, str] | None = None) -> bool:
    """Telemetry is opt-IN until v0.43.1 ships the network code.

    The roadmap entry calls this opt-out, but until the privacy policy
    + PostHog wire-up land we keep it default-OFF so no payload is built
    or sent. Users may enable explicitly with `SOUP_TELEMETRY=1`.
    """
    source = env if env is not None else os.environ
    raw = source.get(_TELEMETRY_ENV_VAR)
    if raw is None:
        return False
    val = raw.strip().lower()
    if val in {"1", "true", "yes", "on"}:
        return True
    return False


def get_or_create_distinct_id() -> str:
    """Return the anonymous telemetry UUID, generating and persisting it if needed.

    The identifier is stored at ``~/.soup/telemetry_id``. It contains NO
    user-identifying or hardware-identifying information — it is a random
    UUID4 generated locally for event deduplication.
    All filesystem operations fail soft so telemetry can NEVER crash training.
    """
    import uuid  # noqa: PLC0415
    from pathlib import Path  # noqa: PLC0415

    from soup_cli.utils.constants import SOUP_DIR  # noqa: PLC0415

    id_file = Path.home() / SOUP_DIR / "telemetry_id"
    try:
        if id_file.exists():
            saved = id_file.read_text(encoding="utf-8").strip()
            # Validate format strictly: must be a valid UUID
            if str(uuid.UUID(saved)) == saved.lower():
                return saved
    except Exception:
        pass

    new_id = str(uuid.uuid4())
    try:
        id_file.parent.mkdir(parents=True, exist_ok=True)
        id_file.write_text(new_id + "\n", encoding="utf-8")
    except Exception:
        pass
    return new_id


def build_telemetry_payload(
    *,
    soup_version: str,
    command: str,
    duration_seconds: float | int | None = None,
    distinct_id: str | None = None,
) -> dict:
    """Build the hardware-info-only telemetry payload.

    The payload contains NO user data, dataset paths, model names, or
    config contents. Documented schema:

      - `soup_version`: caller-supplied
      - `command`: top-level CLI command (e.g. `train`, `data ingest`)
      - `python`: major.minor only
      - `os`: platform.system()
      - `arch`: platform.machine()
      - `duration_seconds`: optional, finite float / int / None
      - `distinct_id`: anonymous persistent UUID (or ephemeral in-memory
        UUID when telemetry is disabled, avoiding filesystem side effects)

    Raises ValueError for non-string `command` / `soup_version` and for
    non-finite `duration_seconds`.
    """
    if not isinstance(soup_version, str) or not soup_version:
        raise ValueError("soup_version must be a non-empty string")
    if "\x00" in soup_version:
        raise ValueError("soup_version must not contain null bytes")
    if not isinstance(command, str) or not command:
        raise ValueError("command must be a non-empty string")
    if "\x00" in command:
        raise ValueError("command must not contain null bytes")
    if duration_seconds is not None:
        # bool is a subclass of int — reject explicitly (project policy)
        if isinstance(duration_seconds, bool) or not isinstance(
            duration_seconds, (int, float)
        ):
            raise ValueError("duration_seconds must be int / float / None")
        if not math.isfinite(float(duration_seconds)):
            raise ValueError("duration_seconds must be finite")
        if duration_seconds < 0:
            raise ValueError("duration_seconds must be >= 0")

    if distinct_id is not None:
        resolved_distinct_id = str(distinct_id)
    elif is_telemetry_enabled():
        resolved_distinct_id = get_or_create_distinct_id()
    else:
        import uuid  # noqa: PLC0415

        resolved_distinct_id = str(uuid.uuid4())

    py = platform.python_version_tuple()
    py_major_minor = f"{py[0]}.{py[1]}"
    return {
        "soup_version": soup_version,
        "command": command,
        "python": py_major_minor,
        "os": platform.system(),
        "arch": platform.machine(),
        "duration_seconds": (
            float(duration_seconds) if duration_seconds is not None else None
        ),
        "distinct_id": resolved_distinct_id,
    }


# v0.53.8 #90 — PostHog telemetry live wiring.
# Opt-IN via SOUP_TELEMETRY=1; silent-fail on any network/transport error
# so telemetry can NEVER crash training. 1s hard timeout, HTTPS-only.

_POSTHOG_HOST = "https://us.i.posthog.com"
_POSTHOG_ENDPOINT = f"{_POSTHOG_HOST}/i/v0/e/"
# v0.53.10 #154 — bundled public write-only project key for Soup CLI
# telemetry. The key is INTENTIONALLY hard-coded: PostHog "phc_*" keys are
# write-only (cannot read events back); rotating it requires a release.
# Operators wanting to point telemetry at their own PostHog project should
# set ``SOUP_POSTHOG_KEY`` AND ``SOUP_POSTHOG_ENDPOINT`` together; both env
# vars are validated by :func:`_resolve_posthog_target`.
_POSTHOG_DEFAULT_KEY = "phc_soup_public_write_only"
_TELEMETRY_TIMEOUT_S = 1.0


def is_placeholder_posthog_key(key: object) -> bool:
    """Return True if ``key`` is the unprovisioned placeholder key."""
    if not isinstance(key, str) or not key:
        return True
    clean = key.strip()
    return clean == _POSTHOG_DEFAULT_KEY or clean.startswith("phc_soup_public_write_only")


# Sentinel for "caller did not pass an endpoint, fall back to default + env".
_POSTHOG_ENDPOINT_DEFAULT = object()


def _resolve_posthog_target(
    api_key: str | None,
    endpoint: object = _POSTHOG_ENDPOINT_DEFAULT,
    env: dict[str, str] | None = None,
) -> tuple[str, str] | None:
    """Resolve ``(key, endpoint)`` from explicit args + ``SOUP_POSTHOG_*`` env.

    v0.53.10 #154 — adds env-var overrides for the bundled defaults so users
    on private PostHog instances can point Soup telemetry at their own
    project without a code change. Precedence:

    1. Explicit ``api_key`` / ``endpoint`` kwargs (caller wins).
    2. ``SOUP_POSTHOG_KEY`` env var (overrides ``_POSTHOG_DEFAULT_KEY``).
    3. ``SOUP_POSTHOG_ENDPOINT`` env var (overrides
       ``_POSTHOG_ENDPOINT``; must be HTTPS + pass the v0.51.0 SSRF policy).
    4. Bundled defaults.

    Returns ``None`` when any input fails validation (silent no-op so
    telemetry can never crash training).
    """
    import os  # noqa: PLC0415 — local lazy import

    src = env if env is not None else os.environ
    # Endpoint resolution: explicit caller > env override > default.
    # Use a sentinel default so a caller who passes
    # ``endpoint=_POSTHOG_ENDPOINT`` (locking in the default) is NOT silently
    # overridden by ``SOUP_POSTHOG_ENDPOINT`` (code-review HIGH fix).
    if endpoint is _POSTHOG_ENDPOINT_DEFAULT:
        env_endpoint = src.get("SOUP_POSTHOG_ENDPOINT")
        resolved_endpoint = env_endpoint or _POSTHOG_ENDPOINT
    else:
        resolved_endpoint = endpoint
    if not isinstance(resolved_endpoint, str):
        return None
    if not _telemetry_endpoint_is_safe(resolved_endpoint):
        return None
    # Key resolution: explicit caller > env override > default.
    if api_key is not None:
        key = api_key
    else:
        key = src.get("SOUP_POSTHOG_KEY") or _POSTHOG_DEFAULT_KEY
    if not isinstance(key, str) or not key:
        return None
    # Reject control chars / whitespace in the key — defends against an
    # operator dropping ``\nAuthorization:...`` into SOUP_POSTHOG_KEY.
    if "\x00" in key or any(ord(c) < 0x20 for c in key) or len(key) > 256:
        return None
    return key, resolved_endpoint


# Internal / non-routable TLD suffixes rejected without DNS resolution (#599).
_INTERNAL_TLD_SUFFIXES: tuple[str, ...] = (
    ".local",
    ".internal",
    ".localhost",
    ".lan",
    ".home",
    ".corp",
    ".intranet",
)

_INTERNAL_TLD_EXACT: frozenset[str] = frozenset({
    "local",
    "internal",
    "localhost",
    "lan",
    "home",
    "corp",
    "intranet",
})


def _is_trusted_posthog_domain(host: str) -> bool:
    """Return True if ``host`` belongs to the trusted PostHog domain hierarchy."""
    return host == "posthog.com" or host.endswith(".posthog.com")


def _resolve_host_ips(host: str, *, timeout: float = 1.0) -> list[str] | None:
    """Resolve ``host`` to IP addresses within ``timeout`` seconds.

    Returns a list of IP strings, or ``None`` on resolution failure,
    resolver error, or timeout (fail-closed).
    """
    import socket  # noqa: PLC0415
    import threading  # noqa: PLC0415

    result: list[str] = []
    error: list[Exception] = []

    def _worker() -> None:
        try:
            info = socket.getaddrinfo(host, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
            for item in info:
                sockaddr = item[4]
                ip = sockaddr[0]
                if ip:
                    result.append(ip)
        except Exception as exc:
            error.append(exc)

    thread = threading.Thread(target=_worker, daemon=True)
    thread.start()
    thread.join(timeout=timeout)
    if thread.is_alive() or error or not result:
        return None
    return result


def _telemetry_endpoint_is_safe(endpoint: str) -> bool:
    """Tiered endpoint guard for telemetry destinations (#593, #599).

    Three-tier validation architecture:

    1. **Primary Control (Trusted Allowlist):** Canonical PostHog domains
       (``us.i.posthog.com``, ``eu.i.posthog.com``, ``*.posthog.com``) are
       accepted immediately. This guarantees sub-millisecond validation with
       zero network requests and zero risk of resolver timeout on default paths.
    2. **Static Syntactic & Suffix Guards:** For custom endpoint URLs, reject
       loopback names (``localhost``), literal private/link-local/loopback IPs
       (including abbreviated/hex/octal IPv4 forms), and internal TLD suffixes
       (``.local``, ``.internal``, ``.lan``, ``.home``, ``.corp``, etc.) without
       performing network lookups.
    3. **Defence-in-Depth DNS Resolution:** For custom non-default FQDNs, resolve
       the host to IP addresses with a bounded timeout and fail closed. Rejects if
       any resolved address is a loopback, private, or link-local IP. This raises
       the bar against hostname indirection (e.g. ``10.0.0.1.nip.io``,
       ``localtest.me``). Note: this does not prevent DNS rebinding across separate
       lookups, but provides defence in depth for custom endpoint configurations.

    The HTTPS-only requirement (``startswith("https://")``) is applied first.
    """
    if not isinstance(endpoint, str) or not endpoint.startswith("https://"):
        return False

    # Layer 1: baseline sanitization (CRLF, null bytes, types, 0.0.0.0).
    try:
        from soup_cli.utils.hubs import validate_hub_endpoint

        validate_hub_endpoint(endpoint, hub="telemetry")
    except (TypeError, ValueError):
        return False

    from urllib.parse import urlparse  # noqa: PLC0415

    from soup_cli.utils.hubs import (  # noqa: PLC0415
        _LOOPBACK_HOSTS,
        _is_private_or_link_local,
    )

    host = (urlparse(endpoint).hostname or "").lower().rstrip(".")
    if not host:
        return False

    # Tier 1: Primary Control — Trusted PostHog domain allowlist.
    # Eliminates DNS lookups for default configurations (99%+ of runs).
    if _is_trusted_posthog_domain(host):
        return True

    # Tier 2: Static rejection (literal IPs, loopback hosts, internal TLDs).
    if host in _LOOPBACK_HOSTS:
        return False
    if _is_private_or_link_local(host):
        return False
    if host in _INTERNAL_TLD_EXACT or host.endswith(_INTERNAL_TLD_SUFFIXES):
        return False

    # Tier 3: Defence-in-depth DNS resolution for custom non-default FQDNs.
    # Raises the bar against hostname indirection; fails closed on error/timeout.
    resolved_ips = _resolve_host_ips(host)
    if not resolved_ips:
        return False

    for ip in resolved_ips:
        clean_ip = ip.rstrip(".")
        if clean_ip in _LOOPBACK_HOSTS or _is_private_or_link_local(clean_ip):
            return False

    return True


def send_telemetry_payload(
    payload: dict[str, object],
    *,
    api_key: str | None = None,
    timeout: float = _TELEMETRY_TIMEOUT_S,
    endpoint: object = _POSTHOG_ENDPOINT_DEFAULT,
) -> bool:
    """POST ``payload`` to PostHog if telemetry is enabled, else no-op.

    Returns ``True`` on a 2xx response, ``False`` on any failure or skip.
    NEVER raises — telemetry is best-effort and must never crash training.

    Args:
        payload: dict built by :func:`build_telemetry_payload`. Required keys
            are validated upstream by the builder.
        api_key: PostHog project key. Defaults to the bundled write-only key.
        timeout: socket connect and read timeout in seconds (default 1.0 s;
            DNS resolution excluded).
        endpoint: full PostHog capture URL (must be HTTPS).
    """
    if not is_telemetry_enabled():
        return False
    if not isinstance(payload, dict) or not payload:
        return False
    if isinstance(timeout, bool) or not isinstance(timeout, (int, float)):
        return False
    if not math.isfinite(float(timeout)) or timeout <= 0:
        return False
    # v0.53.10 #154 — resolve key + endpoint via env-override-aware helper.
    # Returns ``None`` when either input fails validation; treat as silent
    # no-op so telemetry remains best-effort.
    resolved = _resolve_posthog_target(api_key, endpoint)
    if resolved is None:
        return False
    key, endpoint = resolved
    if is_placeholder_posthog_key(key):
        import sys  # noqa: PLC0415

        try:
            sys.stderr.write("Notice: telemetry is not yet live.\n")
        except Exception:
            pass
        return False

    body = {
        "api_key": key,
        "event": payload.get("command", "soup_event"),
        "properties": {k: v for k, v in payload.items() if k != "command"},
    }
    try:
        import json  # noqa: PLC0415
        from urllib import request  # noqa: PLC0415

        encoded = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode(
            "utf-8"
        )
        req = request.Request(
            endpoint,
            data=encoded,
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        with request.urlopen(req, timeout=float(timeout)) as response:
            status = getattr(response, "status", None)
            if not isinstance(status, int):
                status = response.getcode()
            return isinstance(status, int) and 200 <= status < 300
    except Exception:  # noqa: BLE001 — telemetry must never crash training
        return False


# v0.53.8 #89 — Friendly missing-dep panel for HF Trainer `--tracker`.
# When user passes `--tracker mlflow` without mlflow installed, HF raises
# a generic ImportError mid-training; this helper lets the CLI surface a
# pip-install advisory BEFORE construction.


def tracker_missing_dep_message(name: str) -> str | None:
    """Return a friendly install advisory for ``name`` if the package is
    missing, else None.

    Always returns ``None`` for `wandb` / `tensorboard` / `none` (the
    legacy backends), since those are part of the standard HF Trainer
    extra and not part of v0.43.0's additive set.
    """
    if not isinstance(name, str):
        return None
    canonical = name.lower()
    if canonical not in NEW_TRACKERS_V0_43:
        return None
    pkg = required_tracker_package(canonical)
    if not pkg:
        return None
    # Use ``importlib.util.find_spec`` (non-executing probe) so we don't
    # incur side effects from the tracker's top-level module (e.g. swanlab
    # initialises network threads on import). ``sys.modules[pkg] = None``
    # raises ``ValueError`` on find_spec — treat that as missing too so
    # tests can simulate the absent-package path without subprocess.
    import importlib.util
    import sys

    sentinel = object()
    cached = sys.modules.get(pkg, sentinel)
    if cached is None:
        missing = True
    elif cached is not sentinel:
        # Module is already imported (or test injected a real-shaped mock).
        missing = False
    else:
        try:
            missing = importlib.util.find_spec(pkg) is None
        except (ImportError, ValueError):
            missing = True
    if missing:
        return (
            f"--tracker {canonical} requires the '{pkg}' package. "
            f"Install with: pip install soup-cli[trackers] "
            f"(or pip install {pkg})"
        )
    return None


def resolve_report_to(
    *,
    wandb: bool = False,
    tensorboard: bool = False,
    tracker: str | None = None,
) -> str:
    """Resolve the HF Trainer `report_to` value from CLI flags + --tracker.

    Mutual-exclusion: only one of (wandb, tensorboard, tracker) may be set.
    Empty string / None on `tracker` is treated as unset.
    """
    set_count = sum(
        1
        for x in (
            bool(wandb),
            bool(tensorboard),
            bool(tracker) if isinstance(tracker, str) and tracker else False,
        )
        if x
    )
    if set_count > 1:
        raise ValueError(
            "--wandb, --tensorboard, and --tracker are mutually exclusive"
        )
    if wandb:
        return "wandb"
    if tensorboard:
        return "tensorboard"
    if tracker:
        return validate_tracker_name(tracker)
    return "none"