Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

27.6 KB
Raw Blame
"""v0.69.0 Part B — Expectations suite for chat data.

Great-Expectations-flavoured assertions over JSONL rows. Each expectation is a
pure function returning a frozen ``ExpectationResult``; the suite runner
composes them and returns a ``SuiteReport`` whose ``passed`` flag is the
AND of every contained expectation. CI gates use exit code 3 on failure
(matches v0.55 / v0.56 / v0.64 / v0.65 gate convention).

Composes with:
  - v0.47.0 ``data_score.detect_pii`` (PII regex / Presidio backend)
  - v0.56.0 ``diagnose.refusal.looks_like_refusal`` (English refusal phrases)
  - v0.19.0 judge backends (operator-injected callable for judge expectations)
"""

from __future__ import annotations

import difflib
import math
import os
from dataclasses import dataclass
from typing import Any, Callable, List, Mapping, Optional, Sequence, Tuple

from soup_cli.utils.row_text import (
    extract_assistant_text as _extract_assistant_text,
)
from soup_cli.utils.row_text import (
    extract_row_parts as _extract_row_parts,
)
from soup_cli.utils.row_text import (
    extract_row_text as _extract_row_text,
)

# DoS caps + bounds for validators.
_MAX_NAME_LEN = 128
_MAX_DETAILS_PER_RESULT = 32
_MAX_DETAIL_LEN = 256
_MAX_SUITE_LEN = 64
_MAX_FILE_BYTES = 1_048_576  # 1 MiB
_MIN_TOKEN_BOUND = 1
_MAX_TOKEN_BOUND = 1_048_576

# Closed allowlist — the 4 expectation kinds the v0.69.0 plan calls out — mapped to
# the arguments each one takes and their defaults. #1428: a suite key outside this
# table is refused at parse time instead of running the expectation on its defaults.
_EXPECTATION_ARGS: dict[str, dict[str, Any]] = {
    "expect_no_pii": {},
    "expect_token_length_between": {"min_tokens": 1, "max_tokens": _MAX_TOKEN_BOUND},
    "expect_no_refusal_pattern": {},
    "expect_chosen_preferred_over_rejected_by_judge": {
        "threshold": 0.7,
        "judge": None,
        "advisory": False,
    },
}
_SUPPORTED_EXPECTATIONS = tuple(_EXPECTATION_ARGS)
SUPPORTED_EXPECTATIONS: frozenset = frozenset(_SUPPORTED_EXPECTATIONS)
_ENTRY_KEYS = ("name", "args")

# JudgeFn signature: row mapping in, [0,1] score out (1.0 = chosen wins).
JudgeFn = Callable[[Mapping[str, Any]], float]


def _extract_row_text_for_judge(val: Any) -> str:
    if isinstance(val, str):
        return val
    if isinstance(val, list):
        parts = []
        for item in val:
            if isinstance(item, Mapping) and "content" in item:
                parts.append(str(item["content"]))
            elif isinstance(item, str):
                parts.append(item)
        return "\n".join(parts)
    return str(val) if val is not None else ""


def build_pairwise_judge_fn(judge: Any) -> JudgeFn:
    """Build a JudgeFn from a judge URL string, PairwiseJudge instance, or callable."""
    if callable(judge) and not hasattr(judge, "compare_pair"):
        return judge
    if isinstance(judge, str):
        if not judge.strip():
            raise ValueError("judge URL must be non-empty")
        from soup_cli.eval.gate import _parse_judge_url
        from soup_cli.eval.judge import JudgeEvaluator

        provider, model, api_base = _parse_judge_url(judge)
        evaluator = JudgeEvaluator(
            provider=provider,
            model=model,
            api_base=api_base,
        )
    elif hasattr(judge, "compare_pair"):
        evaluator = judge
    else:
        raise TypeError("judge must be a URL string, PairwiseJudge instance, or callable")

    from soup_cli.eval.judge import pairwise_compare

    def pairwise_judge_fn(row: Mapping[str, Any]) -> float:
        prompt = _extract_row_text_for_judge(row.get("prompt"))
        resp_a = _extract_row_text_for_judge(row.get("chosen"))
        resp_b = _extract_row_text_for_judge(row.get("rejected"))
        verdict = pairwise_compare(prompt, resp_a, resp_b, evaluator, swap=True)
        if verdict == 0:
            return 1.0
        if verdict == 1:
            return 0.0
        return 0.5

    return pairwise_judge_fn


@dataclass(frozen=True)
class ExpectationResult:
    """Outcome of one expectation."""

    name: str
    passed: bool
    num_rows_checked: int
    num_violations: int
    details: Tuple[str, ...]

    def __post_init__(self) -> None:
        validate_expectation_name(self.name)
        if not isinstance(self.passed, bool):
            raise TypeError("ExpectationResult.passed must be bool")
        for field_name, val in (
            ("num_rows_checked", self.num_rows_checked),
            ("num_violations", self.num_violations),
        ):
            if isinstance(val, bool) or not isinstance(val, int):
                raise TypeError(f"ExpectationResult.{field_name} must be int")
            if val < 0:
                raise ValueError(
                    f"ExpectationResult.{field_name} must be non-negative"
                )
        if not isinstance(self.details, tuple):
            raise TypeError(
                "ExpectationResult.details must be a tuple (frozen=True does "
                "not make List immutable)"
            )


@dataclass(frozen=True)
class ExpectationSpec:
    """One row of a suite YAML."""

    name: str
    args: Mapping[str, Any]


@dataclass(frozen=True)
class SuiteSpec:
    """Validated expectations suite."""

    expectations: Tuple[ExpectationSpec, ...]


@dataclass(frozen=True)
class SuiteReport:
    """Outcome of a full suite run."""

    passed: bool
    results: Tuple[ExpectationResult, ...]


# -----------------------------------------------------------------------------
# Validators
# -----------------------------------------------------------------------------


def validate_expectation_name(name: object) -> str:
    """Return the canonical lower-case expectation name."""
    if isinstance(name, bool) or not isinstance(name, str):
        raise TypeError(
            f"expectation name must be str, got {type(name).__name__}"
        )
    if not name:
        raise ValueError("expectation name must be non-empty")
    if "\x00" in name:
        raise ValueError("expectation name must not contain null bytes")
    if len(name) > _MAX_NAME_LEN:
        raise ValueError(
            f"expectation name must be <= {_MAX_NAME_LEN} chars"
        )
    canonical = name.strip().lower()
    if canonical not in SUPPORTED_EXPECTATIONS:
        raise ValueError(
            f"unknown expectation: {name!r}. "
            f"supported: {sorted(SUPPORTED_EXPECTATIONS)}"
        )
    return canonical


def _check_threshold(value: object, *, field: str) -> float:
    if isinstance(value, bool):
        raise TypeError(f"{field} must be float, not bool")
    if not isinstance(value, (int, float)):
        raise TypeError(f"{field} must be a number")
    fval = float(value)
    if not math.isfinite(fval):
        raise ValueError(f"{field} must be finite")
    if not (0.0 <= fval <= 1.0):
        raise ValueError(f"{field} must be in [0.0, 1.0]")
    return fval


def _check_token_bound(value: object, *, field: str) -> int:
    if isinstance(value, bool):
        raise TypeError(f"{field} must be int, not bool")
    if not isinstance(value, int):
        raise TypeError(f"{field} must be an integer")
    if value < _MIN_TOKEN_BOUND or value > _MAX_TOKEN_BOUND:
        raise ValueError(
            f"{field} must be in [{_MIN_TOKEN_BOUND}, {_MAX_TOKEN_BOUND}]"
        )
    return value


def _check_token_bounds(min_tokens: object, max_tokens: object) -> Tuple[int, int]:
    low = _check_token_bound(min_tokens, field="min_tokens")
    high = _check_token_bound(max_tokens, field="max_tokens")
    if low > high:
        raise ValueError(
            f"min_tokens ({low}) must be <= max_tokens ({high})"
        )
    return low, high


# -----------------------------------------------------------------------------
# Row helpers & Format Extraction Rules
# -----------------------------------------------------------------------------


FORMAT_EXTRACTION_RULES: Mapping[str, Mapping[str, Tuple[str, ...]]] = {
    "alpaca": {
        "text_fields": ("instruction", "input", "output", "system"),
        "assistant_fields": ("output", "response"),
    },
    "sharegpt": {
        "text_fields": ("conversations",),
        "assistant_fields": ("conversations",),
    },
    "chatml": {
        "text_fields": ("messages", "tools", "tool_calls"),
        "assistant_fields": ("messages", "tool_calls"),
    },
    "dpo": {
        "text_fields": ("prompt", "chosen", "rejected"),
        "assistant_fields": ("chosen",),
    },
    "kto": {
        "text_fields": ("prompt", "completion"),
        "assistant_fields": ("completion",),
    },
    "llava": {
        "text_fields": ("conversations",),
        "assistant_fields": ("conversations",),
    },
    "sharegpt4v": {
        "text_fields": ("conversations",),
        "assistant_fields": ("conversations",),
    },
    "embedding": {
        "text_fields": ("anchor", "positive", "negative"),
        "assistant_fields": (),
    },
    "audio": {
        "text_fields": ("messages",),
        "assistant_fields": ("messages",),
    },
    "asr": {
        "text_fields": ("text",),
        "assistant_fields": ("text",),
    },
    "plaintext": {
        "text_fields": ("text",),
        "assistant_fields": ("text",),
    },
    "tool-calling": {
        "text_fields": ("messages", "tools", "tool_calls"),
        "assistant_fields": ("messages", "tool_calls"),
    },
    "raft": {
        "text_fields": ("query", "golden_doc", "distractor_docs", "answer"),
        "assistant_fields": ("answer",),
    },
    "prm": {
        "text_fields": ("prompt", "completions"),
        "assistant_fields": ("completions",),
    },
    "input_output": {
        "text_fields": ("segments",),
        "assistant_fields": ("segments",),
    },
}


def _check_rows(rows: object) -> Sequence[Mapping[str, Any]]:
    if isinstance(rows, str) or isinstance(rows, bytes):
        raise TypeError("rows must be a list of mappings, not a string")
    if not hasattr(rows, "__iter__"):
        raise TypeError("rows must be iterable")
    try:
        materialised = list(rows)
    except TypeError as exc:
        raise TypeError(f"rows is not iterable: {exc}") from exc
    return materialised


def _truncate_detail(detail: str) -> str:
    if len(detail) > _MAX_DETAIL_LEN:
        return detail[: _MAX_DETAIL_LEN - 3] + "..."
    return detail


# -----------------------------------------------------------------------------
# Expectations
# -----------------------------------------------------------------------------


def expect_no_pii(rows: Any) -> ExpectationResult:
    """Fail when any row contains an email / phone / SSN / credit-card hit.

    Reuses v0.47.0 ``data_score.detect_pii`` (Presidio backend if available;
    falls back to the in-tree 4-regex baseline) — so improvements there
    immediately flow into expectations.
    """
    materialised = _check_rows(rows)
    from soup_cli.utils.data_score import detect_pii  # lazy

    num_violations = 0
    details: List[str] = []
    for index, row in enumerate(materialised):
        if not isinstance(row, Mapping):
            details.append(_truncate_detail(f"rows[{index}]: not a dict"))
            num_violations += 1
            continue
        parts = _extract_row_parts(row)
        if not parts or not any(p.strip() for p in parts):
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(
                    _truncate_detail(f"rows[{index}]: no extractable text")
                )
            continue
        all_hits = []
        for part in parts:
            if not part:
                continue
            try:
                hits = detect_pii(part)
                if hits:
                    all_hits.extend(hits)
            except (TypeError, ValueError):
                continue
        if all_hits:
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                kinds = sorted({hit.get("kind", "?") for hit in all_hits})
                details.append(
                    _truncate_detail(
                        f"rows[{index}]: PII detected ({', '.join(kinds)})"
                    )
                )
    return ExpectationResult(
        name="expect_no_pii",
        passed=num_violations == 0,
        num_rows_checked=len(materialised),
        num_violations=num_violations,
        details=tuple(details),
    )


def expect_token_length_between(
    rows: Any,
    *,
    min_tokens: int,
    max_tokens: int,
) -> ExpectationResult:
    """Fail when any row's token count (whitespace-split) is out of bounds."""
    low, high = _check_token_bounds(min_tokens, max_tokens)
    materialised = _check_rows(rows)
    num_violations = 0
    details: List[str] = []
    for index, row in enumerate(materialised):
        if not isinstance(row, Mapping):
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(_truncate_detail(f"rows[{index}]: not a dict"))
            continue
        text = _extract_row_text(row)
        if not text or not text.strip():
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(
                    _truncate_detail(f"rows[{index}]: no extractable text")
                )
            continue
        token_count = len(text.split())
        if token_count < low or token_count > high:
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(
                    _truncate_detail(
                        f"rows[{index}]: {token_count} tokens (want [{low}, {high}])"
                    )
                )
    return ExpectationResult(
        name="expect_token_length_between",
        passed=num_violations == 0,
        num_rows_checked=len(materialised),
        num_violations=num_violations,
        details=tuple(details),
    )


def expect_no_refusal_pattern(rows: Any) -> ExpectationResult:
    """Fail when any assistant-side output matches the v0.56.0 refusal regex."""
    materialised = _check_rows(rows)
    from soup_cli.utils.diagnose.refusal import looks_like_refusal  # lazy

    num_violations = 0
    details: List[str] = []
    for index, row in enumerate(materialised):
        if not isinstance(row, Mapping):
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(_truncate_detail(f"rows[{index}]: not a dict"))
            continue
        full_text = _extract_row_text(row)
        if not full_text or not full_text.strip():
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(
                    _truncate_detail(f"rows[{index}]: no extractable text")
                )
            continue
        text = _extract_assistant_text(row)
        if not text or not text.strip():
            continue
        if looks_like_refusal(text):
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(
                    _truncate_detail(f"rows[{index}]: refusal pattern matched")
                )
    return ExpectationResult(
        name="expect_no_refusal_pattern",
        passed=num_violations == 0,
        num_rows_checked=len(materialised),
        num_violations=num_violations,
        details=tuple(details),
    )


def expect_chosen_preferred_over_rejected_by_judge(
    rows: Any,
    *,
    judge_fn: Optional[JudgeFn] = None,
    threshold: float = 0.7,
    advisory: bool = False,
) -> ExpectationResult:
    """Fail when ``judge_fn(row) < threshold`` on a preference row.

    Rows must carry both ``chosen`` and ``rejected``. ``judge_fn`` returns a
    score in [0, 1]; 1.0 means the judge fully prefers chosen over rejected.

    When ``judge_fn`` is omitted and ``advisory=False``, every preference row
    is flagged as a violation for missing judge configuration (#1433).
    To explicitly trust existing labels without running a judge, set
    ``advisory=True``.
    """
    t = _check_threshold(threshold, field="threshold")
    if not isinstance(advisory, bool):
        raise TypeError("advisory must be bool")
    if judge_fn is not None and not callable(judge_fn):
        raise TypeError("judge_fn must be callable or None")
    materialised = _check_rows(rows)
    num_violations = 0
    details: List[str] = []
    for index, row in enumerate(materialised):
        if not isinstance(row, Mapping):
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(_truncate_detail(f"rows[{index}]: not a dict"))
            continue
        if "chosen" not in row or "rejected" not in row:
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(
                    _truncate_detail(
                        f"rows[{index}]: missing chosen/rejected field"
                    )
                )
            continue
        if judge_fn is None:
            if advisory:
                score: float = 1.0
            else:
                num_violations += 1
                if len(details) < _MAX_DETAILS_PER_RESULT:
                    details.append(
                        _truncate_detail(
                            f"rows[{index}]: no judge configured: set args.judge "
                            "(e.g. ollama://llama3.1) or pass --judge, or remove this expectation"
                        )
                    )
                continue
        else:
            try:
                raw = judge_fn(row)
            except Exception as exc:  # noqa: BLE001 - one bad row mustn't crash the suite
                from soup_cli.eval.judge import JudgeUnavailableError

                if isinstance(exc, JudgeUnavailableError):
                    raise
                num_violations += 1
                if len(details) < _MAX_DETAILS_PER_RESULT:
                    details.append(
                        _truncate_detail(f"rows[{index}]: judge raised")
                    )
                continue
            if isinstance(raw, bool) or not isinstance(raw, (int, float)):
                num_violations += 1
                if len(details) < _MAX_DETAILS_PER_RESULT:
                    details.append(
                        _truncate_detail(
                            f"rows[{index}]: judge returned non-number"
                        )
                    )
                continue
            score = float(raw)
            if not math.isfinite(score):
                num_violations += 1
                continue
        if score < t:
            num_violations += 1
            if len(details) < _MAX_DETAILS_PER_RESULT:
                details.append(
                    _truncate_detail(
                        f"rows[{index}]: judge score {score:.3f} < {t:.3f}"
                    )
                )
    if judge_fn is None and advisory and not details:
        details.append(f"advisory: no judge ran; {len(materialised)} rows not scored")
    return ExpectationResult(
        name="expect_chosen_preferred_over_rejected_by_judge",
        passed=num_violations == 0,
        num_rows_checked=len(materialised),
        num_violations=num_violations,
        details=tuple(details),
    )


# -----------------------------------------------------------------------------
# Suite parsing + execution
# -----------------------------------------------------------------------------


def _key_repr(key: object) -> str:
    # repr() spells control bytes out, so a hostile key cannot reach the terminal raw.
    return repr(str(key)[:_MAX_NAME_LEN])


_MAX_KEYS_NAMED = 10


def _key_list(keys: Sequence[object]) -> str:
    # Ten keys and a count: a 1 MiB suite can carry tens of thousands of unknown keys,
    # and naming them all made the message nearly as large as the file.
    shown = ", ".join(map(_key_repr, keys[:_MAX_KEYS_NAMED]))
    rest = len(keys) - _MAX_KEYS_NAMED
    return f"{shown} and {rest} more" if rest > 0 else shown


def _validate_entry_keys(index: int, name: str, entry: Mapping[str, Any]) -> None:
    extra = [key for key in entry if key not in _ENTRY_KEYS]
    if not extra:
        return
    misplaced = [key for key in extra if key in _EXPECTATION_ARGS[name]]
    if misplaced:
        raise ValueError(
            f"expectations[{index}]: {_key_list(misplaced)} "
            f"must go under 'args:' for {name}, not beside 'name'"
        )
    raise ValueError(
        f"expectations[{index}]: unknown key(s) {_key_list(extra)}; "
        "an entry takes only 'name' and 'args'"
    )


def _validate_args(index: int, name: str, args: Mapping[str, Any]) -> None:
    accepted = _EXPECTATION_ARGS[name]
    unknown = [key for key in args if key not in accepted]
    if unknown:
        shown = _key_repr(unknown[0])
        if not accepted:
            raise ValueError(f"expectations[{index}]: {name} takes no arguments, got {shown}")
        close = difflib.get_close_matches(str(unknown[0]), sorted(accepted), n=1)
        hint = f" (did you mean {close[0]!r}?)" if close else ""
        raise ValueError(
            f"expectations[{index}]: {name} takes no argument {shown}{hint}; "
            f"accepted: {', '.join(sorted(accepted))}"
        )
    merged = {**accepted, **args}
    if name == "expect_token_length_between":
        _check_token_bounds(merged["min_tokens"], merged["max_tokens"])
    elif name == "expect_chosen_preferred_over_rejected_by_judge":
        _check_threshold(merged["threshold"], field="threshold")
        if merged.get("judge") is not None:
            if not isinstance(merged["judge"], str):
                raise TypeError(f"expectations[{index}]: judge must be a string")
            if not merged["judge"].strip():
                raise ValueError(f"expectations[{index}]: judge must be non-empty")
            from soup_cli.eval.gate import _parse_judge_url

            try:
                _parse_judge_url(merged["judge"])
            except ValueError as exc:
                raise ValueError(f"expectations[{index}]: {exc}") from exc
        if not isinstance(merged.get("advisory", False), bool):
            raise TypeError(f"expectations[{index}]: advisory must be a boolean")


def parse_suite_spec(raw: Any) -> SuiteSpec:
    """Validate a suite dict and return a ``SuiteSpec``."""
    if not isinstance(raw, dict):
        raise TypeError("suite spec must be a dict")
    extra = [key for key in raw if key != "expectations"]
    if extra:  # #1483: a top-level key is a typo or an option that does not exist
        raise ValueError(
            f"unknown top-level key(s) {_key_list(extra)}; "
            "a suite takes only 'expectations'"
        )
    raw_expectations = raw.get("expectations")
    if raw_expectations is None:
        raise ValueError("suite must define 'expectations' key")
    if not isinstance(raw_expectations, list):
        raise TypeError("suite 'expectations' must be a list")
    if not raw_expectations:
        raise ValueError("suite 'expectations' must be a non-empty list")
    if len(raw_expectations) > _MAX_SUITE_LEN:
        raise ValueError(
            f"suite 'expectations' exceeds {_MAX_SUITE_LEN} entries"
        )

    items: List[ExpectationSpec] = []
    for index, entry in enumerate(raw_expectations):
        if not isinstance(entry, dict):
            raise TypeError(f"expectations[{index}] must be a dict")
        name = validate_expectation_name(entry.get("name", ""))
        _validate_entry_keys(index, name, entry)
        args = entry.get("args", {})
        if not isinstance(args, dict):
            raise TypeError(f"expectations[{index}].args must be a dict")
        _validate_args(index, name, args)
        items.append(ExpectationSpec(name=name, args=dict(args)))
    return SuiteSpec(expectations=tuple(items))


def parse_suite_yaml(text: object) -> SuiteSpec:
    """Parse a suite YAML string into a validated ``SuiteSpec``."""
    if not isinstance(text, str):
        raise TypeError("suite text must be a string")
    if "\x00" in text:
        raise ValueError("suite text must not contain null bytes")
    if len(text.encode("utf-8")) > _MAX_FILE_BYTES:
        raise ValueError(f"suite text exceeds {_MAX_FILE_BYTES} bytes")
    import yaml  # lazy

    from soup_cli.utils.yaml_limits import load_yaml_limited

    try:
        data = load_yaml_limited(text, "suite text", max_bytes=_MAX_FILE_BYTES)
    except yaml.YAMLError as exc:
        raise ValueError(f"invalid YAML: {exc}") from exc
    return parse_suite_spec(data)


def load_suite_yaml(path: object) -> SuiteSpec:
    """Load a suite YAML from a path under cwd (with TOCTOU symlink reject).

    Delegates to ``utils.paths.enforce_under_cwd_and_no_symlink`` (v0.59.0
    centralised helper).
    """
    from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink

    if isinstance(path, bool) or not isinstance(path, str):
        raise TypeError("path must be a string")
    enforce_under_cwd_and_no_symlink(path, "suite path")
    if not os.path.lexists(path):
        raise FileNotFoundError(f"suite file not found: {path!r}")
    real = os.path.realpath(path)
    if not os.path.isfile(real):
        raise FileNotFoundError(f"suite file not found: {path!r}")
    if os.path.getsize(real) > _MAX_FILE_BYTES:
        raise ValueError(f"suite file exceeds {_MAX_FILE_BYTES} bytes")
    with open(real, "r", encoding="utf-8") as handle:
        return parse_suite_yaml(handle.read())


def _dispatch_expectation(
    spec: ExpectationSpec,
    rows: Sequence[Mapping[str, Any]],
    *,
    judge_fn: Optional[JudgeFn] = None,
) -> ExpectationResult:
    """Dispatch one ``ExpectationSpec`` against ``rows``.

    Per code-review HIGH H2: passes ``args`` values through *as-is* — the
    expectation function's own validators must reject bool / NaN / Inf /
    type-coerced inputs. This prevents an ``int("5")`` silent coercion from
    bypassing ``_check_token_bound`` bool-rejection.
    """
    name = spec.name
    args = {**_EXPECTATION_ARGS.get(name, {}), **spec.args}
    if name == "expect_no_pii":
        return expect_no_pii(rows)
    if name == "expect_token_length_between":
        return expect_token_length_between(
            rows,
            min_tokens=args["min_tokens"],
            max_tokens=args["max_tokens"],
        )
    if name == "expect_no_refusal_pattern":
        return expect_no_refusal_pattern(rows)
    if name == "expect_chosen_preferred_over_rejected_by_judge":
        effective_judge = judge_fn
        if effective_judge is None and args.get("judge"):
            effective_judge = build_pairwise_judge_fn(args["judge"])
        return expect_chosen_preferred_over_rejected_by_judge(
            rows,
            judge_fn=effective_judge,
            threshold=args["threshold"],
            advisory=args["advisory"],
        )
    raise ValueError(f"unhandled expectation: {name!r}")  # pragma: no cover


def run_suite(
    rows: Any,
    spec: SuiteSpec,
    *,
    judge_fn: Optional[JudgeFn] = None,
) -> SuiteReport:
    """Run every expectation in ``spec`` against ``rows``."""
    if not isinstance(spec, SuiteSpec):
        raise TypeError("spec must be a SuiteSpec")
    materialised = _check_rows(rows)
    results: List[ExpectationResult] = []
    for expectation in spec.expectations:
        results.append(
            _dispatch_expectation(expectation, materialised, judge_fn=judge_fn)
        )
    passed = all(r.passed for r in results)
    return SuiteReport(passed=passed, results=tuple(results))


__all__ = [
    "ExpectationResult",
    "ExpectationSpec",
    "FORMAT_EXTRACTION_RULES",
    "JudgeFn",
    "SUPPORTED_EXPECTATIONS",
    "SuiteReport",
    "SuiteSpec",
    "build_pairwise_judge_fn",
    "expect_chosen_preferred_over_rejected_by_judge",
    "expect_no_pii",
    "expect_no_refusal_pattern",
    "expect_token_length_between",
    "load_suite_yaml",
    "parse_suite_spec",
    "parse_suite_yaml",
    "run_suite",
    "validate_expectation_name",
]