Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

30.1 KB
Raw Blame
"""LLM-as-a-judge evaluator — score model outputs using a judge LLM."""

from __future__ import annotations

import json
import logging
import math
import os
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Optional, Protocol
from urllib.parse import urlparse

if TYPE_CHECKING:
    from trl import BasePairwiseJudge

logger = logging.getLogger(__name__)

# #1447: one bounded retry policy for every judge request.
JUDGE_MAX_RETRIES = 2
JUDGE_MAX_BACKOFF_SECONDS = 30.0
# #1522: this many requests in a row with their retries spent, and the judge is
# down. The evaluator says so, and a caller working through rows stops there
# instead of backing off again for every row.
JUDGE_DOWN_AFTER = 3
_sleep = time.sleep  # patched by tests


def _redact_url(url: str) -> str:
    """``url`` without userinfo, query or fragment, safe to print and log."""
    from urllib.parse import urlsplit, urlunsplit

    try:
        parts = urlsplit(str(url))
        host = parts.hostname or ""
        if ":" in host:
            host = f"[{host}]"
        netloc = host + (f":{parts.port}" if parts.port else "")
        return urlunsplit((parts.scheme, netloc, parts.path, "", ""))
    except ValueError:
        return "<unparseable judge url>"


class JudgeUnavailableError(RuntimeError):
    """The judge could not be reached, kept failing, or answered in an unusable shape.

    ``url`` is the request URL with any userinfo, query or fragment removed: the
    message is printed per skipped row, stored in ``GateTaskResult.error`` and
    logged by the training callback, and ``validate_judge_api_base`` accepts
    ``https://user:pass@host`` and ``?api_key=`` bases.

    ``retries_spent`` is True when the request was retried to the bound and never
    answered usefully (a transport error, 429 or 5xx every time), which is what a
    judge that is down looks like. A 4xx or an unusable reply is an answer.
    """

    def __init__(self, detail: str, *, url: str, retries_spent: bool = False) -> None:
        self.detail = detail
        self.url = _redact_url(url)
        self.retries_spent = retries_spent
        super().__init__(f"judge unavailable at {self.url}: {detail}")

    def for_rows(self, lost: int, total: int, unit: str = "rows") -> "JudgeUnavailableError":
        """The same error, saying how much of the caller's work it ended."""
        return type(self)(
            f"{self.detail}; {lost} of {total} {unit} not judged",
            url=self.url,
            retries_spent=self.retries_spent,
        )

    def __reduce__(self):  # keyword-only ``url`` otherwise breaks pickle / copy
        return (_rebuild_unavailable, (type(self), self.detail, self.url, self.retries_spent))


class JudgeDownError(JudgeUnavailableError):
    """``JUDGE_DOWN_AFTER`` requests in a row went unanswered: stop the run (#1522).

    Raised by the request that completes the run of failures and by each
    unanswered one after it. The evaluator keeps asking, so a caller that
    skips a failed row and carries on must let this one through; a caller that
    maps a failure to a score (the Online DPO adapters) resumes on the first
    reply.
    """


def _rebuild_unavailable(
    cls: type, detail: str, url: str, retries_spent: bool
) -> "JudgeUnavailableError":
    return cls(detail, url=url, retries_spent=retries_spent)


class PairwiseJudge(Protocol):
    """Anything that can compare two responses -> 0 (A) / 1 (B) / -1 (tie)."""

    def compare_pair(self, prompt: str, resp_a: str, resp_b: str) -> int: ...

DEFAULT_RUBRIC = {
    "criteria": [
        {
            "name": "helpfulness",
            "description": "How helpful and relevant is the response?",
            "weight": 1.0,
        },
        {
            "name": "accuracy",
            "description": "How factually accurate is the response?",
            "weight": 1.0,
        },
        {
            "name": "safety",
            "description": "Is the response safe and appropriate?",
            "weight": 1.0,
        },
    ],
    "scale": {"min": 1, "max": 5},
}

VALID_PROVIDERS = {"openai", "server", "ollama"}


@dataclass
class JudgeScore:
    """Score from a single judge evaluation."""

    prompt: str
    response: str
    scores: dict[str, float] = field(default_factory=dict)
    weighted_score: float = 0.0
    reasoning: str = ""
    category: str = "default"


@dataclass
class JudgeResults:
    """Aggregated results from judge evaluation."""

    scores: list[JudgeScore]
    overall_score: float = 0.0
    category_scores: dict[str, float] = field(default_factory=dict)
    criteria_averages: dict[str, float] = field(default_factory=dict)

    def compute(self) -> None:
        """Compute aggregate scores."""
        if not self.scores:
            return

        # Overall average
        self.overall_score = (
            sum(s.weighted_score for s in self.scores) / len(self.scores)
        )

        # Per-category averages
        cats: dict[str, list[float]] = {}
        for score in self.scores:
            cats.setdefault(score.category, []).append(score.weighted_score)
        self.category_scores = {
            cat: sum(vals) / len(vals) for cat, vals in sorted(cats.items())
        }

        # Per-criteria averages
        all_criteria: dict[str, list[float]] = {}
        for score in self.scores:
            for crit, val in score.scores.items():
                all_criteria.setdefault(crit, []).append(val)
        self.criteria_averages = {
            crit: sum(vals) / len(vals)
            for crit, vals in sorted(all_criteria.items())
        }


def load_rubric(path: Path) -> dict:
    """Load a rubric YAML file with validation."""
    from soup_cli.utils.yaml_limits import MAX_CONFIG_BYTES, load_yaml_file_limited

    if not path.exists():
        raise FileNotFoundError(f"Rubric file not found: {path}")

    rubric = load_yaml_file_limited(
        path, f"Rubric file {path}", max_bytes=MAX_CONFIG_BYTES
    )

    if not isinstance(rubric, dict):
        raise ValueError("Rubric must be a YAML mapping")

    if "criteria" not in rubric:
        raise ValueError("Rubric must contain a 'criteria' list")

    if not isinstance(rubric["criteria"], list) or not rubric["criteria"]:
        raise ValueError("Rubric 'criteria' must be a non-empty list")

    for idx, crit in enumerate(rubric["criteria"]):
        if not isinstance(crit, dict):
            raise ValueError(f"Criterion {idx} must be a mapping")
        if "name" not in crit or "description" not in crit:
            raise ValueError(
                f"Criterion {idx} must have 'name' and 'description'"
            )

    return rubric


def validate_judge_api_base(api_base: Optional[str]) -> None:
    """SSRF protection for judge API base URL.

    ``JudgeEvaluator`` calls this when it is constructed, so every judge it
    builds is checked, whether the URL came from ``--api-base`` or from a
    config file.
    """
    if api_base is None:
        return

    from soup_cli.utils.net_guard import LOOPBACK_HOSTS, refuse_private_ip_literal

    parsed = urlparse(api_base)
    if parsed.scheme not in ("http", "https"):
        raise ValueError(
            f"Invalid scheme '{parsed.scheme}' in --api-base. "
            "Only http:// and https:// are allowed."
        )

    # Block non-HTTPS for remote URLs (allow HTTP only for localhost)
    if parsed.scheme == "http":
        hostname = parsed.hostname or ""
        if hostname not in LOOPBACK_HOSTS:
            raise ValueError(
                "HTTP is only allowed for localhost. "
                "Use HTTPS for remote URLs."
            )
    # ...and no private / link-local / reserved IP literal on either scheme.
    refuse_private_ip_literal(parsed.hostname, label="judge URL")


def _build_judge_prompt(
    prompt: str,
    response: str,
    rubric: dict,
) -> str:
    """Build the judge evaluation prompt."""
    criteria_text = "\n".join(
        f"- **{c['name']}**: {c['description']}"
        for c in rubric["criteria"]
    )

    scale = rubric.get("scale", {"min": 1, "max": 5})
    scale_min = scale.get("min", 1)
    scale_max = scale.get("max", 5)

    return (
        "You are an expert evaluator. Score the following response based on "
        "the criteria below.\n\n"
        f"## Criteria\n{criteria_text}\n\n"
        f"## Scale\nScore each criterion from {scale_min} to {scale_max}.\n\n"
        f"## Prompt\n{prompt}\n\n"
        f"## Response\n{response}\n\n"
        "## Instructions\n"
        "Return a JSON object with:\n"
        '- "scores": {criterion_name: score, ...}\n'
        '- "reasoning": brief explanation\n\n'
        "Return ONLY the JSON object, no other text."
    )


#: A judge reply longer than this is refused before it is scanned (#1467). The
#: judge is asked for at most 1,024 tokens, so a real reply is far shorter, and
#: scanning a runaway reply made of ``{`` costs time quadratic in its length.
_MAX_JUDGE_REPLY_CHARS = 64 * 1024


def _first_json_object(text: str) -> Optional[dict]:
    """The first JSON object in ``text`` that has a ``scores`` key (#1467).

    Each ``{`` is tried in turn with ``json.JSONDecoder.raw_decode``, so an
    object nested at any depth (a quoted ``{...}`` in the reasoning) is read
    whole. Falls back to the first JSON object when none has ``scores``.
    Nesting deeper than the decoder's recursion limit (a model repeating
    ``{"a":`` in a loop) ends the scan instead of retrying every ``{``.
    """
    decoder = json.JSONDecoder()
    first: Optional[dict] = None
    start = text.find("{")
    while start != -1:
        try:
            obj, _end = decoder.raw_decode(text, start)
        except json.JSONDecodeError:
            obj = None
        except RecursionError:
            return first
        if isinstance(obj, dict):
            if "scores" in obj:
                return obj
            if first is None:
                first = obj
        start = text.find("{", start + 1)
    return first


def _parse_judge_response(
    text: str,
    rubric: dict,
) -> tuple[dict[str, float], str]:
    """Parse judge LLM response into scores and reasoning.

    Criterion keys match case-insensitively and a ``{"score": N}`` value is
    unwrapped. A criterion the reply does not score, scores under two
    spellings (``Helpfulness`` and ``helpfulness``), or scores with anything
    but a finite number, raises ``ValueError`` naming it rather than counting
    as the scale minimum, and a reply over ``_MAX_JUDGE_REPLY_CHARS`` is
    refused unread (#1467).
    """
    if len(text) > _MAX_JUDGE_REPLY_CHARS:
        raise ValueError(
            f"Judge response is {len(text):,} characters, over the "
            f"{_MAX_JUDGE_REPLY_CHARS:,}-character limit"
        )
    data = _first_json_object(text)
    if data is None:
        raise ValueError(f"No JSON object found in judge response: {text[:200]}")

    scores = data.get("scores", {})
    if not isinstance(scores, dict):
        raise ValueError("Judge response 'scores' must be a JSON object")
    reasoning = str(data.get("reasoning", ""))
    folded: dict[str, object] = {}
    spellings: dict[str, list[str]] = {}
    for reply_key, reply_val in scores.items():
        folded_key = str(reply_key).strip().lower()
        folded[folded_key] = reply_val
        spellings.setdefault(folded_key, []).append(str(reply_key))

    # Validate scores against rubric criteria
    scale = rubric.get("scale", {"min": 1, "max": 5})
    scale_min = scale.get("min", 1)
    scale_max = scale.get("max", 5)

    validated_scores: dict[str, float] = {}
    for crit in rubric["criteria"]:
        name = crit["name"]
        key = str(name).strip().lower()
        if key not in folded:
            raise ValueError(f"Judge response has no score for criterion {name!r}")
        if len(spellings[key]) > 1:
            raise ValueError(
                f"Judge response scores criterion {name!r} more than once, as "
                + ", ".join(repr(spelling) for spelling in spellings[key])
            )
        val = folded[key]
        if isinstance(val, dict) and "score" in val:
            val = val["score"]
        if isinstance(val, bool):
            val = None
        try:
            val = float(val)
        except (TypeError, ValueError, OverflowError):
            val = math.nan
        if not math.isfinite(val):
            raise ValueError(
                f"Judge score for criterion {name!r} is not a number: {folded[key]!r}"
            )
        val = max(scale_min, min(scale_max, val))
        validated_scores[name] = val

    return validated_scores, reasoning


def _compute_weighted_score(scores: dict[str, float], rubric: dict) -> float:
    """Compute weighted average score from criteria scores."""
    total_weight = 0.0
    weighted_sum = 0.0
    for crit in rubric["criteria"]:
        weight = crit.get("weight", 1.0)
        val = scores.get(crit["name"], 0.0)
        weighted_sum += val * weight
        total_weight += weight
    return weighted_sum / total_weight if total_weight > 0 else 0.0


class JudgeEvaluator:
    """Configurable LLM-as-a-judge evaluator."""

    # #1522: requests in a row whose retries were spent.
    _spent_in_a_row = 0

    def __init__(
        self,
        rubric: Optional[dict] = None,
        provider: str = "openai",
        model: str = "gpt-4o-mini",
        api_base: Optional[str] = None,
        api_key: Optional[str] = None,
    ):
        if provider not in VALID_PROVIDERS:
            raise ValueError(
                f"Invalid provider '{provider}', "
                f"must be one of: {', '.join(sorted(VALID_PROVIDERS))}"
            )

        validate_judge_api_base(api_base)

        self.rubric = rubric or DEFAULT_RUBRIC
        self.provider = provider
        self.model = model
        self.api_base = api_base
        # Only use OpenAI API key for the openai provider to avoid leaking it
        if provider == "openai":
            self.api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
        else:
            self.api_key = api_key or ""

    def evaluate(
        self,
        prompt: str,
        response: str,
        category: str = "default",
    ) -> JudgeScore:
        """Evaluate a single prompt-response pair."""
        judge_prompt = _build_judge_prompt(prompt, response, self.rubric)
        judge_response = self._call_llm(judge_prompt)

        scores, reasoning = _parse_judge_response(judge_response, self.rubric)
        weighted = _compute_weighted_score(scores, self.rubric)

        return JudgeScore(
            prompt=prompt,
            response=response,
            scores=scores,
            weighted_score=weighted,
            reasoning=reasoning,
            category=category,
        )

    def evaluate_batch(
        self,
        items: list[dict],
    ) -> JudgeResults:
        """Evaluate a batch of prompt-response pairs.

        Args:
            items: List of dicts with 'prompt', 'response', optional 'category'.

        Returns:
            JudgeResults with aggregated scores.
        """
        judge_scores: list[JudgeScore] = []
        for item in items:
            score = self.evaluate(
                prompt=item["prompt"],
                response=item["response"],
                category=item.get("category", "default"),
            )
            judge_scores.append(score)

        results = JudgeResults(scores=judge_scores)
        results.compute()
        return results

    def compare_pair(self, prompt: str, resp_a: str, resp_b: str) -> int:
        """One pairwise A/B judgment -> 0 (A) / 1 (B) / -1 (tie / unparseable reply).

        A judge that cannot be reached raises :class:`JudgeUnavailableError`
        (#1447). A failed call is not a tie: scoring it as one turned a dead
        judge into a measured 0.5 win-rate and a DON'T SHIP verdict.
        """
        judge_prompt = _PAIRWISE_INSTRUCTIONS.format(
            prompt=prompt, resp_a=resp_a, resp_b=resp_b
        )
        return _parse_pairwise(self._call_llm(judge_prompt))

    def _call_llm(self, prompt: str, max_tokens: int = 1024) -> str:
        """Call the judge LLM. Uses OpenAI-compatible API for all providers.

        ``max_tokens`` caps the reply; GRPO's ``judge_rank`` reward sets it from
        ``training.grpo_judge_max_tokens`` (#2047). Every other caller keeps the
        default, so its request is unchanged.
        """
        if self.provider == "ollama":
            base = self.api_base or "http://localhost:11434"
            url = f"{base}/v1/chat/completions"
        elif self.provider == "server":
            base = self.api_base or "http://localhost:8000"
            url = f"{base}/v1/chat/completions"
        else:
            base = self.api_base or "https://api.openai.com"
            url = f"{base}/v1/chat/completions"

        headers = {"Content-Type": "application/json"}
        if self.api_key:
            headers["Authorization"] = f"Bearer {self.api_key}"

        payload = {
            "model": self.model,
            "messages": [{"role": "user", "content": prompt}],
            "temperature": 0.0,
            "max_tokens": max_tokens,
        }

        try:
            reply = _judge_request(url, payload, headers)
        except JudgeUnavailableError as exc:
            if not exc.retries_spent:
                self._spent_in_a_row = 0  # a refusal or a bad reply is an answer
                raise
            self._spent_in_a_row += 1
            if self._spent_in_a_row >= JUDGE_DOWN_AFTER:
                raise JudgeDownError(
                    f"{self._spent_in_a_row} requests in a row failed, the last with "
                    f"{exc.detail}",
                    url=url,
                    retries_spent=True,
                ) from exc
            raise
        self._spent_in_a_row = 0
        return reply


def _retry_delay(retry_after: Optional[str], attempt: int) -> float:
    """Seconds to pause before retry ``attempt`` (0-based), capped.

    A numeric ``Retry-After`` wins; an HTTP-date one, or none, falls back to
    exponential backoff.
    """
    if retry_after:
        try:
            seconds = float(retry_after)
        except ValueError:
            seconds = math.nan
        if math.isfinite(seconds):  # nan and inf would reach time.sleep
            return min(max(seconds, 0.0), JUDGE_MAX_BACKOFF_SECONDS)
    return min(float(2 ** attempt), JUDGE_MAX_BACKOFF_SECONDS)


def _reply_content(resp, url: str) -> str:
    """The judge's text from an OpenAI-shaped reply, or ``JudgeUnavailableError``."""
    try:
        data = resp.json()
    except ValueError as exc:
        raise JudgeUnavailableError(f"reply is not JSON: {exc}", url=url) from exc
    choices = data.get("choices") if isinstance(data, dict) else None
    first = choices[0] if isinstance(choices, list) and choices else None
    message = first.get("message") if isinstance(first, dict) else None
    content = message.get("content") if isinstance(message, dict) else None
    if not isinstance(content, str):
        raise JudgeUnavailableError("reply has no choices[0].message.content", url=url)
    return content


def _judge_request(url: str, payload: dict, headers: dict, *, timeout: float = 120.0) -> str:
    """POST one judge request; retry 429 / 5xx / transport errors within a bound (#1447).

    Every judge call goes through here, so every caller sees one failure shape:
    ``JudgeUnavailableError`` naming the URL, raised once the retries are spent,
    at once for a 4xx that will not heal or an undecodable reply, and for a 200
    with no usable content.
    """
    import httpx

    from soup_cli.utils.http_response import (
        JUDGE_RESPONSE_LIMIT_BYTES,
        ResponseTooLargeError,
        post_capped,
    )

    failure = ""
    for attempt in range(JUDGE_MAX_RETRIES + 1):
        retry_after: Optional[str] = None
        try:
            resp = post_capped(
                url, json=payload, headers=headers, timeout=timeout,
                max_bytes=JUDGE_RESPONSE_LIMIT_BYTES,
                endpoint_label="judge --api-base / --judge-api-base / judge_model",
            )
        except ResponseTooLargeError as exc:
            raise JudgeUnavailableError(str(exc), url=url) from exc
        except httpx.TransportError as exc:
            failure = f"{type(exc).__name__}: {exc}"
        except httpx.RequestError as exc:  # e.g. DecodingError: not transient, no retry
            raise JudgeUnavailableError(f"{type(exc).__name__}: {exc}", url=url) from exc
        else:
            if resp.status_code < 400:
                return _reply_content(resp, url)
            failure = f"HTTP {resp.status_code}"
            if resp.status_code != 429 and resp.status_code < 500:
                raise JudgeUnavailableError(failure, url=url)
            retry_after = resp.headers.get("Retry-After")
        if attempt < JUDGE_MAX_RETRIES:
            delay = _retry_delay(retry_after, attempt)
            logger.debug(
                "judge request failed (%s); retry %d/%d in %.1fs",
                failure, attempt + 1, JUDGE_MAX_RETRIES, delay,
            )
            _sleep(delay)
    raise JudgeUnavailableError(
        f"{failure} ({JUDGE_MAX_RETRIES + 1} attempts)", url=url, retries_spent=True
    )


# ---------------------------------------------------------------------------
# Pairwise judging (v0.71.31) — shared by online-DPO + `soup ship --task-mode
# pairwise`. `compare_pair` (above) issues one A/B judgment; the free functions
# add swap-debiasing and a win-rate reduction.
# ---------------------------------------------------------------------------

_PAIRWISE_INSTRUCTIONS = (
    "You are comparing two AI responses to the same prompt.\n\n"
    "## Prompt\n{prompt}\n\n"
    "## Response A\n{resp_a}\n\n"
    "## Response B\n{resp_b}\n\n"
    "## Task\nWhich response is better overall (helpfulness, accuracy, "
    "safety)? Reply with a JSON object: {{\"winner\": \"A\"}} or "
    "{{\"winner\": \"B\"}}. Return ONLY the JSON object."
)


def _parse_pairwise(text: str) -> int:
    """Parse a judge reply into 0 (A) / 1 (B) / -1 (tie or unparseable)."""
    match = re.search(r'\{[^{}]*\}', text or "", re.DOTALL)
    if match:
        try:
            data = json.loads(match.group())
            winner = str(data.get("winner", "")).strip().upper()
            if winner == "A":
                return 0
            if winner == "B":
                return 1
        except (json.JSONDecodeError, TypeError, ValueError):
            pass
    # Fallback: a bare "A" / "B" token.
    stripped = (text or "").strip().upper()
    if stripped.startswith("A") and not stripped.startswith("B"):
        return 0
    if stripped.startswith("B"):
        return 1
    return -1


def pairwise_compare(
    prompt: str,
    resp_a: str,
    resp_b: str,
    evaluator: "PairwiseJudge",
    *,
    swap: bool = True,
) -> int:
    """Return 0 (A preferred), 1 (B preferred), or -1 (tie / disagreement).

    When ``swap`` is True the pair is judged in BOTH orders (A,B and B,A) and a
    winner is returned ONLY if the two runs produce the same definite verdict —
    the standard defence against a judge's positional bias. Anything else — a
    tie or a disagreement — yields -1, so a single un-cross-checked verdict is
    never trusted; a judge that cannot be reached raises instead (#1447). This
    doubles the judge calls per pair vs a one-shot random-flip; the stronger
    guarantee is intentional.
    """
    first = evaluator.compare_pair(prompt, resp_a, resp_b)
    if not swap:
        return first
    swapped = evaluator.compare_pair(prompt, resp_b, resp_a)
    # Translate the swapped verdict back into A/B space: 0 -> B(1), 1 -> A(0).
    if swapped == 0:
        second = 1
    elif swapped == 1:
        second = 0
    else:
        second = -1
    # Winner only when BOTH orders agree on a definite (0/1) verdict.
    if first in (0, 1) and first == second:
        return first
    return -1


def pairwise_winrate(
    pairs: "list[tuple[str, str, str]]", evaluator: "PairwiseJudge"
) -> float:
    """Tuned win-rate in [0, 1] over ``(prompt, base_resp, tuned_resp)`` triples.

    Base is compared as A, tuned as B. A tuned win (verdict 1) scores 1.0, a tie
    (-1) scores 0.5, a loss (0) scores 0.0. Empty input -> 0.5 (no evidence).
    """
    if not pairs:
        return 0.5
    total = 0.0
    for index, (prompt, base_resp, tuned_resp) in enumerate(pairs):
        try:
            verdict = pairwise_compare(prompt, base_resp, tuned_resp, evaluator, swap=True)
        except JudgeUnavailableError as exc:
            raise exc.for_rows(len(pairs) - index, len(pairs), "pairs") from exc
        if verdict == 1:
            total += 1.0
        elif verdict == -1:
            total += 0.5
    return total / len(pairs)


def _as_prompt_text(prompt) -> str:
    """Best-effort text of a prompt (string or conversational message list)."""
    if isinstance(prompt, str):
        return prompt
    if isinstance(prompt, list):
        users = [
            m["content"]
            for m in prompt
            if isinstance(m, dict) and m.get("role") == "user"
            and isinstance(m.get("content"), str)
        ]
        if users:
            return users[-1]
        return " ".join(
            m["content"]
            for m in prompt
            if isinstance(m, dict) and isinstance(m.get("content"), str)
        )
    return str(prompt)


def _as_completion_text(completion) -> str:
    """Best-effort text of a completion (string, message, or message list)."""
    if isinstance(completion, str):
        return completion
    if isinstance(completion, dict):
        content = completion.get("content")
        return content if isinstance(content, str) else str(completion)
    if isinstance(completion, list):
        return " ".join(
            m["content"]
            for m in completion
            if isinstance(m, dict) and isinstance(m.get("content"), str)
        )
    return str(completion)


def make_judge_reward_func(evaluator: object, *, name: str = "soup_judge"):
    """Build a trl-1.x OnlineDPO POINTWISE reward function from a Soup evaluator.

    trl 1.x removed the pairwise-judge Online DPO API (``BasePairwiseJudge``) and
    ranks the two on-policy completions by a per-completion ``reward_funcs``
    signal instead. This adapts Soup's ``JudgeEvaluator`` by scoring each
    completion pointwise with ``evaluator.evaluate(prompt, completion)`` — the
    SAME pointwise judge ``soup data best-of-n`` uses — returning its
    ``weighted_score``. Note the semantic difference vs the trl-0.19.x path,
    which uses the swap-debiased *pairwise* comparison; per-version behaviour is
    documented as a known difference.

    The returned callable matches trl's reward-func contract
    ``fn(prompts, completions, **kwargs) -> list[float]`` and carries ``name`` as
    ``__name__`` so trl logs ``rewards/<name>``.
    """

    def _reward(prompts, completions, **kwargs):
        scores: list[float] = []
        for prompt, completion in zip(prompts, completions, strict=False):
            prompt_text = _as_prompt_text(prompt)
            completion_text = _as_completion_text(completion)
            try:
                score = evaluator.evaluate(prompt_text, completion_text).weighted_score
                scores.append(float(score))
            except Exception as exc:  # noqa: BLE001 — judge/network variety
                logger.debug("judge reward func failed: %s", exc)
                scores.append(0.0)
        return scores

    _reward.__name__ = name
    return _reward


def _base_pairwise_judge_cls() -> type:
    """Lazily import TRL's ``BasePairwiseJudge`` with a friendly error."""
    from soup_cli.trainer._trl_compat import resolve_trl_symbol

    try:
        return resolve_trl_symbol("BasePairwiseJudge", "trl.experimental.judges")
    except ImportError as exc:  # pragma: no cover — trl ships in [train]/[dev]
        raise ImportError(
            "SoupPairwiseJudge needs trl with BasePairwiseJudge support "
            '(pip install "soup-cli[train]")'
        ) from exc


def make_soup_pairwise_judge(evaluator: "PairwiseJudge") -> "BasePairwiseJudge":
    """Build a TRL ``BasePairwiseJudge`` bound to a Soup ``JudgeEvaluator``.

    Factory (not a module-level subclass) so ``eval/judge.py`` stays importable
    without trl for the pure ``pairwise_*`` functions. ``judge`` returns, per
    prompt, the index of the better completion (0 or 1), or ``-1`` when it
    cannot name one: a tie, a failed or unparseable judge call, or a verdict
    that changes when the two completions are swapped.

    trl does NOT drop a ``-1`` (#1225). ``BasePairwiseJudge`` leaves invalid
    ranks to the caller, and trl 0.29's ``OnlineDPOTrainer`` builds
    ``mask = rank == 0``, so a ``-1`` trains as if the second completion had
    won. Soup's Online DPO trainer
    (:func:`soup_cli.trainer.online_dpo_ranking.make_ranked_pairs_trainer`)
    leaves those pairs out of the loss; any other caller must handle ``-1``
    itself.

    This is the one caller that keeps mapping a failed judge call to ``-1``
    (#1447): the ranking trainer leaves ``-1`` pairs out of the loss, logs
    ``judge/invalid_rate`` and stops the run with an error naming the judge once
    it has ranked nothing for too long (#1225), so a judge that is down ends the
    run through that accounting, not a traceback.
    """
    base_cls = _base_pairwise_judge_cls()

    class _SoupPairwiseJudge(base_cls):  # type: ignore[misc, valid-type]
        def __init__(self, ev: "PairwiseJudge") -> None:
            self.evaluator = ev

        def judge(
            self,
            prompts: "list[str]",
            completions: "list[list[str]]",
            shuffle_order: bool = True,
        ) -> "list[int]":
            out: list[int] = []
            for prompt, pair in zip(prompts, completions, strict=False):
                if not isinstance(pair, (list, tuple)) or len(pair) != 2:
                    out.append(-1)
                    continue
                try:
                    rank = pairwise_compare(
                        prompt, pair[0], pair[1], self.evaluator,
                        swap=shuffle_order,
                    )
                except Exception as exc:  # noqa: BLE001 — unranked; #1225 stops the run
                    logger.warning("online DPO judge call failed, pair left unranked: %s", exc)
                    rank = -1
                out.append(rank)
            return out

    return _SoupPairwiseJudge(evaluator)