Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

24.7 KB
Raw Blame
"""Eval-Gated Training (v0.26.0 Part B).

Declarative ``evals/gate.yaml`` suites define per-task thresholds; the gate
runs them at epoch boundaries during training (or post-hoc via
``soup eval gate``) and surfaces pass / fail / regression verdicts.
"""

from __future__ import annotations

import json
import logging
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Literal, Mapping, Optional
from urllib.parse import urlparse

from pydantic import BaseModel, Field, field_validator

from soup_cli import __version__
from soup_cli.eval.results import newest_eval_rows
from soup_cli.utils.paths import atomic_write_text, is_under_cwd
from soup_cli.utils.yaml_limits import MAX_CONFIG_BYTES, load_yaml_file_limited

logger = logging.getLogger(__name__)

# Envelope keys for stamped baseline files (#404).
_BASELINE_SCORES_KEY = "scores"
_BASELINE_PROVENANCE_KEY = "provenance"
_STAMP_SOUP_VERSION = "soup_version"
_STAMP_SCORER_REVISION = "scorer_revision"


@dataclass(frozen=True)
class GateTaskResult:
    name: str
    score: Optional[float]
    threshold: float
    baseline: Optional[float]
    delta: Optional[float]
    passed: bool
    error: Optional[str] = None


@dataclass(frozen=True)
class GateResult:
    passed: bool
    regression: bool
    task_results: list[GateTaskResult]


class GateTask(BaseModel):
    """One gate task — e.g. a custom eval + threshold."""

    type: Literal["judge", "custom", "benchmark"] = Field(
        description="Task type: judge | custom | benchmark",
    )
    name: str = Field(description="Task name (used as baseline key)")
    threshold: float = Field(
        ge=0.0, le=1000.0,
        description="Minimum score to pass (scale depends on scorer)",
    )
    # type=custom
    tasks: Optional[str] = Field(
        default=None, description="JSONL file of custom eval tasks",
    )
    scorer: Optional[Literal["exact", "contains", "answer", "regex", "semantic"]] = Field(
        default=None, description="Scorer for type=custom",
    )
    # type=judge
    prompts: Optional[str] = Field(
        default=None, description="JSONL prompts for LLM judge",
    )
    judge_model: Optional[str] = Field(
        default=None, description="Judge model URL (e.g. ollama://llama3.1)",
    )
    # type=benchmark
    benchmark: Optional[str] = Field(
        default=None, description="Registered benchmark id (e.g. mini_mmlu)",
    )

    @field_validator("tasks", "prompts")
    @classmethod
    def _clean_path_fields(cls, value: Optional[str]) -> Optional[str]:
        if value is None:
            return None
        if "\x00" in value:
            raise ValueError("path field contains null byte")
        return value

    @field_validator("judge_model")
    @classmethod
    def _valid_judge_url(cls, value: Optional[str]) -> Optional[str]:
        if value is None:
            return None
        # Allowlist of schemes — SSRF hardening consistent with the project.
        from soup_cli.utils.net_guard import LOOPBACK_HOSTS, refuse_private_ip_literal

        parsed = urlparse(value)

        if parsed.scheme == "ollama":
            return value

        if parsed.scheme == "https":
            # A private IP literal is refused when the suite is parsed, so a
            # suite file cannot name one any more than --judge-model can.
            refuse_private_ip_literal(parsed.hostname, label="judge_model URL")
            return value

        if parsed.scheme == "http" and parsed.hostname in LOOPBACK_HOSTS:
            return value

        raise ValueError(
            f"judge_model URL '{value}' uses disallowed scheme - "
            "use ollama://, https://, or http://localhost"
        )


class EvalSuite(BaseModel):
    """Parsed ``evals/gate.yaml``."""

    suite: str = Field(description="Suite name for display / logs")
    tasks: list[GateTask] = Field(default_factory=list)


def _require_gate_tasks(suite: EvalSuite) -> None:
    if not suite.tasks:
        raise ValueError(
            f"eval-gate suite {suite.suite!r} field 'tasks' must contain at least one task"
        )
    names: set[str] = set()
    for task in suite.tasks:
        if task.name in names:
            raise ValueError(
                f"eval-gate suite {suite.suite!r} repeats task name {task.name!r}"
            )
        names.add(task.name)


def load_suite(path: str) -> EvalSuite:
    """Load and validate an eval suite from disk."""
    suite_path = Path(path)
    if not is_under_cwd(suite_path):
        raise ValueError(
            f"eval-gate suite '{path}' is outside cwd - refusing to load"
        )
    if not suite_path.exists():
        raise FileNotFoundError(f"eval-gate suite not found: {path}")
    data = load_yaml_file_limited(
        suite_path, f"eval-gate suite '{path}'", max_bytes=MAX_CONFIG_BYTES
    ) or {}
    suite = EvalSuite(**data)
    _require_gate_tasks(suite)
    return suite


def current_baseline_stamp() -> dict[str, object]:
    """Soup version + bundled-scorer revision for a baseline provenance stamp."""
    from soup_cli.eval.gate_suites import BUNDLED_SCORER_REVISION

    return {
        _STAMP_SOUP_VERSION: str(__version__),
        _STAMP_SCORER_REVISION: int(BUNDLED_SCORER_REVISION),
    }


def _finite_score(value: object, *, name: str, context: str) -> float:
    """Convert and validate that a score is a finite float (#1865)."""
    try:
        fval = float(value)  # type: ignore[arg-type]
    except OverflowError as exc:
        raise ValueError(
            f"{context}: score for {name!r} is out of range (must be finite)"
        ) from exc
    if not math.isfinite(fval):
        raise ValueError(
            f"{context}: score for {name!r} must be finite"
        )
    return fval


def stamp_baseline_scores(scores: Mapping[str, float]) -> dict[str, object]:
    """Build the stamped baseline envelope (shared writer helper, #404).

    Every Soup producer of a ``--baseline`` JSON file must go through this
    helper (or :func:`write_baseline_file`) so the stamp cannot drift between
    commands.
    """
    if not isinstance(scores, Mapping):
        raise TypeError(f"scores must be a mapping, got {type(scores).__name__}")
    clean: dict[str, float] = {}
    for key, value in scores.items():
        name = str(key)
        if not name or "\x00" in name:
            raise ValueError("baseline score names must be non-empty and null-free")
        if isinstance(value, bool) or not isinstance(value, (int, float)):
            raise TypeError(
                f"baseline score for {name!r} must be a number, "
                f"got {type(value).__name__}"
            )
        clean[name] = _finite_score(value, name=name, context="baseline score")
    return {
        _BASELINE_SCORES_KEY: clean,
        _BASELINE_PROVENANCE_KEY: current_baseline_stamp(),
    }


def write_baseline_file(path: str, scores: Mapping[str, float]) -> str:
    """Atomically write a stamped baseline JSON file. Returns the path.

    Refuses an empty score map so a stub / failed gate run cannot mint a
    stamped baseline that looks authoritative (#404 / #485). Reading an old
    empty baseline file remains silent via :func:`resolve_baseline`.
    """
    if not isinstance(scores, Mapping):
        raise TypeError(f"scores must be a mapping, got {type(scores).__name__}")
    if len(scores) == 0:
        raise ValueError(
            "refusing to write an empty baseline (no scored tasks); "
            "re-run with a model that produces scores"
        )
    payload = stamp_baseline_scores(scores)
    return atomic_write_text(
        json.dumps(payload, indent=2, sort_keys=True) + "\n",
        path,
        field="baseline",
    )


def _coerce_score_map(raw: Mapping[object, object], *, context: str) -> dict[str, float]:
    out: dict[str, float] = {}
    for key, value in raw.items():
        name = str(key)
        if not name:
            continue
        if isinstance(value, bool) or not isinstance(value, (int, float)):
            raise ValueError(
                f"{context}: score for {name!r} must be a number, "
                f"got {type(value).__name__}"
            )
        out[name] = _finite_score(value, name=name, context=context)
    return out


def _is_stamped_envelope(data: Mapping[object, object]) -> bool:
    if _BASELINE_SCORES_KEY not in data:
        return False
    if not isinstance(data.get(_BASELINE_SCORES_KEY), Mapping):
        return False
    # Envelope keys only — a flat map that happens to include a "scores"
    # numeric entry stays legacy.
    return set(data.keys()) <= {_BASELINE_SCORES_KEY, _BASELINE_PROVENANCE_KEY}


def _envelope_extra_keys(data: Mapping[object, object]) -> list[str]:
    """Extra top-level keys on a scores+provenance-shaped payload."""
    if _BASELINE_SCORES_KEY not in data:
        return []
    if not isinstance(data.get(_BASELINE_SCORES_KEY), Mapping):
        return []
    return sorted(
        str(k)
        for k in data.keys()
        if k not in {_BASELINE_SCORES_KEY, _BASELINE_PROVENANCE_KEY}
    )


def _provenance_warning(provenance: object) -> Optional[str]:
    """Return a single warning message, or None when the stamp matches."""
    from soup_cli.eval.gate_suites import BUNDLED_SCORER_REVISION

    if not isinstance(provenance, Mapping):
        return (
            "--baseline has unknown provenance (no scorer/version stamp). "
            "Re-measure the baseline with the current Soup so both sides share "
            "one scorer scale."
        )
    raw_rev = provenance.get(_STAMP_SCORER_REVISION)
    if raw_rev is None:
        return (
            "--baseline has unknown provenance (no scorer_revision). "
            "Re-measure the baseline with the current Soup so both sides share "
            "one scorer scale."
        )
    if isinstance(raw_rev, bool) or not isinstance(raw_rev, int):
        return (
            "--baseline provenance scorer_revision is malformed; treating it as "
            "unknown provenance. Re-measure the baseline."
        )
    if int(raw_rev) != int(BUNDLED_SCORER_REVISION):
        return (
            f"--baseline scorer_revision={int(raw_rev)} does not match the "
            f"running Soup (scorer_revision={int(BUNDLED_SCORER_REVISION)}). "
            "The stored scores may be on a different scale — re-measure the "
            "baseline, or drop mismatched names from it to force a live base run."
        )
    return None


def _emit_baseline_warning(
    message: str,
    *,
    warn: Optional[Callable[[str], None]],
) -> None:
    if warn is not None:
        warn(message)
        return
    logger.warning("%s", message)


def _row_provenance(row: Mapping[str, object]) -> Optional[dict[str, object]]:
    """Best-effort stamp from one eval_results row's ``details_json``."""
    raw = row.get("details_json")
    if not raw:
        return None
    try:
        details = json.loads(raw) if isinstance(raw, str) else raw
    except (TypeError, ValueError, json.JSONDecodeError):
        return None
    if not isinstance(details, Mapping):
        return None
    prov = details.get(_BASELINE_PROVENANCE_KEY)
    if isinstance(prov, Mapping):
        return dict(prov)
    return None


def resolve_baseline(
    spec: Optional[str],
    *,
    warn: Optional[Callable[[str], None]] = None,
) -> dict[str, float]:
    """Resolve a baseline specifier to a ``{task_name: score}`` map.

    - ``None`` / ``""``: return empty map
    - ``registry://<id>``: look up eval_results for that entry via the
      experiment tracker, keyed by benchmark
    - filesystem path: JSON mapping of ``{name: score}`` **or** a stamped
      envelope ``{"scores": {...}, "provenance": {...}}`` (#404)

    Provenance is checked on read. A correctly stamped baseline matching the
    running scorer revision is silent. An unstamped (pre-existing) baseline
    warns about unknown provenance, and a stamp whose ``scorer_revision``
    disagrees warns about the mismatch. A file baseline warns at most once;
    a registry baseline warns once per distinct problem and names the
    affected benchmarks.
    """
    if not spec:
        return {}

    if spec.startswith("registry://"):
        from soup_cli.registry.store import RegistryStore

        ref = spec[len("registry://"):]
        with RegistryStore() as store:
            # ``resolve`` strips the scheme itself; pass the raw ref so the
            # error path below reports what the user typed.
            entry_id = store.resolve(ref)
            if entry_id is None:
                raise ValueError(
                    f"registry baseline not found: {ref} (use `soup registry list`)"
                )
            rows = store.get_eval_results(entry_id)
        newest = {
            row["benchmark"]: row
            for row in newest_eval_rows(rows)
        }
        scores = _coerce_score_map(
            {name: row["score"] for name, row in newest.items()},
            context="registry baseline",
        )
        # Registry rows predate the stamp (or carry it inside details_json).
        # Checked per benchmark: re-measuring one benchmark says nothing about
        # the scale of another that was not re-measured. Benchmarks sharing a
        # problem share one warning, which names them.
        stale: dict[str, list[str]] = {}
        for name, row in newest.items():
            message = _provenance_warning(_row_provenance(row))
            if message is not None:
                stale.setdefault(message, []).append(name)
        for message, names in stale.items():
            _emit_baseline_warning(
                f"{message} Affected benchmark(s): {', '.join(sorted(names))}.",
                warn=warn,
            )
        return scores

    # Filesystem path
    baseline_path = Path(spec)
    if not is_under_cwd(baseline_path):
        raise ValueError(
            f"baseline file '{spec}' is outside cwd - refusing to load"
        )
    if not baseline_path.exists():
        raise FileNotFoundError(f"baseline file not found: {spec}")
    try:
        data = json.loads(baseline_path.read_text(encoding="utf-8"))
    except json.JSONDecodeError as exc:
        raise ValueError(f"invalid JSON in baseline file: {exc}") from exc
    if not isinstance(data, dict):
        raise ValueError(
            f"baseline file must be a JSON object mapping name -> score; "
            f"got {type(data).__name__}"
        )

    # Stamped envelope (exact keys) OR scores-mapping with unsupported extras.
    # Extras warn and are ignored so an unrelated numeric ValueError cannot
    # hide the operator-facing message (#404 review MEDIUM 5).
    if isinstance(data.get(_BASELINE_SCORES_KEY), Mapping):
        extra = _envelope_extra_keys(data)
        if extra and not _is_stamped_envelope(data):
            _emit_baseline_warning(
                "--baseline envelope has unsupported extra key(s): "
                f"{', '.join(extra)}. Keep only 'scores' and 'provenance' "
                "(remove the extras) so the file matches the stamped baseline "
                "contract.",
                warn=warn,
            )
        if _is_stamped_envelope(data) or extra:
            scores = _coerce_score_map(
                data[_BASELINE_SCORES_KEY], context="baseline scores"
            )
            message = _provenance_warning(data.get(_BASELINE_PROVENANCE_KEY))
            if message is not None and scores:
                _emit_baseline_warning(message, warn=warn)
            return scores

    # Legacy flat ``{name: score}`` — unknown provenance (#404).
    scores = _coerce_score_map(data, context="baseline file")
    if scores:
        _emit_baseline_warning(
            "--baseline has unknown provenance (unstamped pre-existing file). "
            "Re-measure the baseline with the current Soup so both sides share "
            "one scorer scale.",
            warn=warn,
        )
    return scores


def _parse_judge_url(judge_model: str) -> tuple[str, str, Optional[str]]:
    """Split a ``judge_model`` URL into ``(provider, model, api_base)``.

    Examples:
      ``ollama://llama3.1`` -> ("ollama", "llama3.1", None)
      ``http://localhost:8000/Qwen2.5`` -> ("server", "Qwen2.5", "http://localhost:8000")
      ``https://api.openai.com/gpt-4o-mini`` -> ("openai", "gpt-4o-mini", "https://api.openai.com")
      ``https://judge.example.com/m`` -> ("server", "m", "https://judge.example.com")
    """

    parsed = urlparse(judge_model)

    if parsed.scheme == "ollama":
        return ("ollama", judge_model[len("ollama://"):], None)

    # One loopback set for every outbound gate (#1548): the shared
    # net_guard.LOOPBACK_HOSTS decides ::1 once, here and in every caller.
    from soup_cli.utils.net_guard import LOOPBACK_HOSTS

    if parsed.scheme == "https":
        # Only the OpenAI API host is given the OpenAI provider (and so
        # OPENAI_API_KEY); any other https judge is an OpenAI-compatible
        # server called without that key.
        if (parsed.hostname or "").lower() == "api.openai.com":
            default_provider = "openai"
        else:
            default_provider = "server"
    elif (
        parsed.scheme == "http"
        and parsed.hostname in LOOPBACK_HOSTS
    ):
        default_provider = "server"
    else:
        raise ValueError(
            f"judge_model '{judge_model}' uses unsupported scheme"
        )

    try:
        base, model = judge_model.rsplit("/", 1)
    except ValueError as exc:
        raise ValueError(
            f"judge_model '{judge_model}' missing model id"
        ) from exc

    if not model:
        raise ValueError(
            f"judge_model '{judge_model}' missing model id"
        )

    return (default_provider, model, base)

def _run_judge_task(
    task: GateTask, generate_fn: Callable[[str], str],
) -> float:
    """Run a type=judge task. Generates a completion per prompt, then asks
    the judge model to score the (prompt, response) pair according to the
    evaluator's rubric. Aggregate score is normalised to [0, 1] via min-max.
    """
    if not task.prompts:
        raise ValueError(f"task '{task.name}' is type=judge but 'prompts' is missing")
    if not task.judge_model:
        raise ValueError(
            f"task '{task.name}' is type=judge but 'judge_model' is missing"
        )

    prompts_path = Path(task.prompts)
    if not is_under_cwd(prompts_path):
        raise ValueError(f"prompts file '{task.prompts}' is outside cwd")
    if not prompts_path.exists():
        raise FileNotFoundError(f"prompts file not found: {task.prompts}")

    from soup_cli.eval.judge import JudgeEvaluator

    provider, model, api_base = _parse_judge_url(task.judge_model)
    evaluator = JudgeEvaluator(provider=provider, model=model, api_base=api_base)

    items: list[dict] = []
    with prompts_path.open("r", encoding="utf-8") as fh:
        for line in fh:
            line = line.strip()
            if not line:
                continue
            try:
                row = json.loads(line)
            except json.JSONDecodeError as exc:
                raise ValueError(
                    f"invalid JSONL in {task.prompts}: {exc}"
                ) from exc
            prompt = row.get("prompt", "")
            response = generate_fn(prompt)
            items.append({
                "prompt": prompt,
                "response": response,
                "category": row.get("category", "default"),
            })

    if not items:
        raise ValueError(f"No judge prompts found in {task.prompts} for task '{task.name}'")

    results = evaluator.evaluate_batch(items)
    # Normalise to [0, 1] using the judge's ACTUAL rubric scale (DEFAULT_RUBRIC
    # is 1-5, not 1-10), via min-max so the rubric floor maps to 0.0.
    scale = evaluator.rubric.get("scale", {}) if isinstance(evaluator.rubric, dict) else {}
    if not isinstance(scale, dict):
        scale = {}

    def _num(value: object, default: float) -> float:
        if isinstance(value, (int, float)) and not isinstance(value, bool):
            return float(value)
        return default

    scale_min = _num(scale.get("min", 1), 1.0)
    scale_max = _num(scale.get("max", 5), 5.0)
    span = scale_max - scale_min

    overall = float(getattr(results, "overall_score", scale_min))
    if span > 0:
        normalized = (overall - scale_min) / span
    else:
        normalized = overall
    return max(0.0, min(1.0, normalized))


def _run_benchmark_task(
    task: GateTask, generate_fn: Callable[[str], str],
) -> float:
    """Run a type=benchmark task using the existing forgetting-mini-benchmark."""
    if not task.benchmark:
        raise ValueError(
            f"task '{task.name}' is type=benchmark but 'benchmark' is missing"
        )
    from soup_cli.eval.forgetting import ForgettingDetector

    score = ForgettingDetector(
        generate_fn=generate_fn,
        benchmark=task.benchmark,
    ).run_baseline()
    return max(0.0, min(1.0, float(score)))


def _run_custom_task(
    task: GateTask, generate_fn: Callable[[str], str],
) -> float:
    """Run a type=custom task and return its aggregate score in [0, 1]."""
    from dataclasses import replace

    from soup_cli.eval.custom import load_eval_tasks, require_single_answer, score_task

    if not task.tasks:
        raise ValueError(f"task '{task.name}' is type=custom but 'tasks' is missing")
    tasks = load_eval_tasks(task.tasks)
    if not tasks:
        return 0.0
    # Override scoring if the suite specified one (EvalTask.scoring field)
    if task.scorer == "answer":
        for row, eval_task in enumerate(tasks, start=1):
            require_single_answer(eval_task.expected, f"task '{task.name}' row {row}")
    if task.scorer is not None:
        tasks = [replace(t, scoring=task.scorer) for t in tasks]
    total = 0.0
    for eval_task in tasks:
        output = generate_fn(eval_task.prompt)
        result = score_task(eval_task, output)
        total += float(result.score)
    return total / len(tasks)


def run_gate(
    suite: EvalSuite,
    *,
    generate_fn: Callable[[str], str],
    baseline: Optional[dict[str, float]] = None,
    regression_threshold: float = 0.05,
) -> GateResult:
    """Run every task in ``suite`` and return an aggregate verdict.

    ``generate_fn`` accepts a prompt and returns the model output. Injecting
    this keeps the gate testable without a live model.
    """
    if baseline is None:
        baseline = {}
    elif not isinstance(baseline, Mapping):
        raise TypeError(f"baseline must be a mapping, got {type(baseline).__name__}")
    for base_name, base_val in baseline.items():
        if isinstance(base_val, bool) or not isinstance(base_val, (int, float)):
            raise TypeError(
                f"baseline score for {base_name!r} must be a number, "
                f"got {type(base_val).__name__}"
            )
        _finite_score(
            base_val,
            name=str(base_name),
            context="baseline score",
        )
    _require_gate_tasks(suite)
    task_results: list[GateTaskResult] = []
    any_regressed = False
    any_failed_threshold = False

    for task in suite.tasks:
        score: Optional[float]
        error: Optional[str] = None
        try:
            if task.type == "custom":
                score = _run_custom_task(task, generate_fn)
            elif task.type == "judge":
                score = _run_judge_task(task, generate_fn)
            elif task.type == "benchmark":
                score = _run_benchmark_task(task, generate_fn)
            else:
                # Pydantic Literal already restricts task.type, so this is a
                # belt-and-braces fallthrough.
                raise ValueError(f"unknown task type: {task.type}")
        except (ValueError, FileNotFoundError, OSError, RuntimeError) as exc:
            score = None
            error = str(exc)
        except Exception as exc:  # noqa: BLE001 — surface as score=None, never silent pass
            score = None
            error = f"{type(exc).__name__}: {exc}"

        if score is None:
            # Failed evaluation never silently passes the gate.
            passed_threshold = False
            base_score = baseline.get(task.name)
            delta = None
            regressed = False
        else:
            passed_threshold = score >= task.threshold
            base_score = baseline.get(task.name)
            delta = None
            regressed = False
            if base_score is not None:
                delta = score - base_score
                if delta < -abs(regression_threshold):
                    regressed = True

        if not passed_threshold:
            any_failed_threshold = True
        if regressed:
            any_regressed = True

        task_results.append(GateTaskResult(
            name=task.name,
            score=score,
            threshold=task.threshold,
            baseline=base_score,
            delta=delta,
            passed=passed_threshold and not regressed,
            error=error,
        ))

    return GateResult(
        passed=not (any_failed_threshold or any_regressed),
        regression=any_regressed,
        task_results=task_results,
    )