Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""LLM-as-a-judge evaluator — score model outputs using a judge LLM."""
from __future__ import annotations
import json
import logging
import math
import os
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Optional, Protocol
from urllib.parse import urlparse
if TYPE_CHECKING:
from trl import BasePairwiseJudge
logger = logging.getLogger(__name__)
# #1447: one bounded retry policy for every judge request.
JUDGE_MAX_RETRIES = 2
JUDGE_MAX_BACKOFF_SECONDS = 30.0
# #1522: this many requests in a row with their retries spent, and the judge is
# down. The evaluator says so, and a caller working through rows stops there
# instead of backing off again for every row.
JUDGE_DOWN_AFTER = 3
_sleep = time.sleep # patched by tests
def _redact_url(url: str) -> str:
"""``url`` without userinfo, query or fragment, safe to print and log."""
from urllib.parse import urlsplit, urlunsplit
try:
parts = urlsplit(str(url))
host = parts.hostname or ""
if ":" in host:
host = f"[{host}]"
netloc = host + (f":{parts.port}" if parts.port else "")
return urlunsplit((parts.scheme, netloc, parts.path, "", ""))
except ValueError:
return "<unparseable judge url>"
class JudgeUnavailableError(RuntimeError):
"""The judge could not be reached, kept failing, or answered in an unusable shape.
``url`` is the request URL with any userinfo, query or fragment removed: the
message is printed per skipped row, stored in ``GateTaskResult.error`` and
logged by the training callback, and ``validate_judge_api_base`` accepts
``https://user:pass@host`` and ``?api_key=`` bases.
``retries_spent`` is True when the request was retried to the bound and never
answered usefully (a transport error, 429 or 5xx every time), which is what a
judge that is down looks like. A 4xx or an unusable reply is an answer.
"""
def __init__(self, detail: str, *, url: str, retries_spent: bool = False) -> None:
self.detail = detail
self.url = _redact_url(url)
self.retries_spent = retries_spent
super().__init__(f"judge unavailable at {self.url}: {detail}")
def for_rows(self, lost: int, total: int, unit: str = "rows") -> "JudgeUnavailableError":
"""The same error, saying how much of the caller's work it ended."""
return type(self)(
f"{self.detail}; {lost} of {total} {unit} not judged",
url=self.url,
retries_spent=self.retries_spent,
)
def __reduce__(self): # keyword-only ``url`` otherwise breaks pickle / copy
return (_rebuild_unavailable, (type(self), self.detail, self.url, self.retries_spent))
class JudgeDownError(JudgeUnavailableError):
"""``JUDGE_DOWN_AFTER`` requests in a row went unanswered: stop the run (#1522).
Raised by the request that completes the run of failures and by each
unanswered one after it. The evaluator keeps asking, so a caller that
skips a failed row and carries on must let this one through; a caller that
maps a failure to a score (the Online DPO adapters) resumes on the first
reply.
"""
def _rebuild_unavailable(
cls: type, detail: str, url: str, retries_spent: bool
) -> "JudgeUnavailableError":
return cls(detail, url=url, retries_spent=retries_spent)
class PairwiseJudge(Protocol):
"""Anything that can compare two responses -> 0 (A) / 1 (B) / -1 (tie)."""
def compare_pair(self, prompt: str, resp_a: str, resp_b: str) -> int: ...
DEFAULT_RUBRIC = {
"criteria": [
{
"name": "helpfulness",
"description": "How helpful and relevant is the response?",
"weight": 1.0,
},
{
"name": "accuracy",
"description": "How factually accurate is the response?",
"weight": 1.0,
},
{
"name": "safety",
"description": "Is the response safe and appropriate?",
"weight": 1.0,
},
],
"scale": {"min": 1, "max": 5},
}
VALID_PROVIDERS = {"openai", "server", "ollama"}
@dataclass
class JudgeScore:
"""Score from a single judge evaluation."""
prompt: str
response: str
scores: dict[str, float] = field(default_factory=dict)
weighted_score: float = 0.0
reasoning: str = ""
category: str = "default"
@dataclass
class JudgeResults:
"""Aggregated results from judge evaluation."""
scores: list[JudgeScore]
overall_score: float = 0.0
category_scores: dict[str, float] = field(default_factory=dict)
criteria_averages: dict[str, float] = field(default_factory=dict)
def compute(self) -> None:
"""Compute aggregate scores."""
if not self.scores:
return
# Overall average
self.overall_score = (
sum(s.weighted_score for s in self.scores) / len(self.scores)
)
# Per-category averages
cats: dict[str, list[float]] = {}
for score in self.scores:
cats.setdefault(score.category, []).append(score.weighted_score)
self.category_scores = {
cat: sum(vals) / len(vals) for cat, vals in sorted(cats.items())
}
# Per-criteria averages
all_criteria: dict[str, list[float]] = {}
for score in self.scores:
for crit, val in score.scores.items():
all_criteria.setdefault(crit, []).append(val)
self.criteria_averages = {
crit: sum(vals) / len(vals)
for crit, vals in sorted(all_criteria.items())
}
def load_rubric(path: Path) -> dict:
"""Load a rubric YAML file with validation."""
from soup_cli.utils.yaml_limits import MAX_CONFIG_BYTES, load_yaml_file_limited
if not path.exists():
raise FileNotFoundError(f"Rubric file not found: {path}")
rubric = load_yaml_file_limited(
path, f"Rubric file {path}", max_bytes=MAX_CONFIG_BYTES
)
if not isinstance(rubric, dict):
raise ValueError("Rubric must be a YAML mapping")
if "criteria" not in rubric:
raise ValueError("Rubric must contain a 'criteria' list")
if not isinstance(rubric["criteria"], list) or not rubric["criteria"]:
raise ValueError("Rubric 'criteria' must be a non-empty list")
for idx, crit in enumerate(rubric["criteria"]):
if not isinstance(crit, dict):
raise ValueError(f"Criterion {idx} must be a mapping")
if "name" not in crit or "description" not in crit:
raise ValueError(
f"Criterion {idx} must have 'name' and 'description'"
)
return rubric
def validate_judge_api_base(api_base: Optional[str]) -> None:
"""SSRF protection for judge API base URL.
``JudgeEvaluator`` calls this when it is constructed, so every judge it
builds is checked, whether the URL came from ``--api-base`` or from a
config file.
"""
if api_base is None:
return
from soup_cli.utils.net_guard import LOOPBACK_HOSTS, refuse_private_ip_literal
parsed = urlparse(api_base)
if parsed.scheme not in ("http", "https"):
raise ValueError(
f"Invalid scheme '{parsed.scheme}' in --api-base. "
"Only http:// and https:// are allowed."
)
# Block non-HTTPS for remote URLs (allow HTTP only for localhost)
if parsed.scheme == "http":
hostname = parsed.hostname or ""
if hostname not in LOOPBACK_HOSTS:
raise ValueError(
"HTTP is only allowed for localhost. "
"Use HTTPS for remote URLs."
)
# ...and no private / link-local / reserved IP literal on either scheme.
refuse_private_ip_literal(parsed.hostname, label="judge URL")
def _build_judge_prompt(
prompt: str,
response: str,
rubric: dict,
) -> str:
"""Build the judge evaluation prompt."""
criteria_text = "\n".join(
f"- **{c['name']}**: {c['description']}"
for c in rubric["criteria"]
)
scale = rubric.get("scale", {"min": 1, "max": 5})
scale_min = scale.get("min", 1)
scale_max = scale.get("max", 5)
return (
"You are an expert evaluator. Score the following response based on "
"the criteria below.\n\n"
f"## Criteria\n{criteria_text}\n\n"
f"## Scale\nScore each criterion from {scale_min} to {scale_max}.\n\n"
f"## Prompt\n{prompt}\n\n"
f"## Response\n{response}\n\n"
"## Instructions\n"
"Return a JSON object with:\n"
'- "scores": {criterion_name: score, ...}\n'
'- "reasoning": brief explanation\n\n'
"Return ONLY the JSON object, no other text."
)
#: A judge reply longer than this is refused before it is scanned (#1467). The
#: judge is asked for at most 1,024 tokens, so a real reply is far shorter, and
#: scanning a runaway reply made of ``{`` costs time quadratic in its length.
_MAX_JUDGE_REPLY_CHARS = 64 * 1024
def _first_json_object(text: str) -> Optional[dict]:
"""The first JSON object in ``text`` that has a ``scores`` key (#1467).
Each ``{`` is tried in turn with ``json.JSONDecoder.raw_decode``, so an
object nested at any depth (a quoted ``{...}`` in the reasoning) is read
whole. Falls back to the first JSON object when none has ``scores``.
Nesting deeper than the decoder's recursion limit (a model repeating
``{"a":`` in a loop) ends the scan instead of retrying every ``{``.
"""
decoder = json.JSONDecoder()
first: Optional[dict] = None
start = text.find("{")
while start != -1:
try:
obj, _end = decoder.raw_decode(text, start)
except json.JSONDecodeError:
obj = None
except RecursionError:
return first
if isinstance(obj, dict):
if "scores" in obj:
return obj
if first is None:
first = obj
start = text.find("{", start + 1)
return first
def _parse_judge_response(
text: str,
rubric: dict,
) -> tuple[dict[str, float], str]:
"""Parse judge LLM response into scores and reasoning.
Criterion keys match case-insensitively and a ``{"score": N}`` value is
unwrapped. A criterion the reply does not score, scores under two
spellings (``Helpfulness`` and ``helpfulness``), or scores with anything
but a finite number, raises ``ValueError`` naming it rather than counting
as the scale minimum, and a reply over ``_MAX_JUDGE_REPLY_CHARS`` is
refused unread (#1467).
"""
if len(text) > _MAX_JUDGE_REPLY_CHARS:
raise ValueError(
f"Judge response is {len(text):,} characters, over the "
f"{_MAX_JUDGE_REPLY_CHARS:,}-character limit"
)
data = _first_json_object(text)
if data is None:
raise ValueError(f"No JSON object found in judge response: {text[:200]}")
scores = data.get("scores", {})
if not isinstance(scores, dict):
raise ValueError("Judge response 'scores' must be a JSON object")
reasoning = str(data.get("reasoning", ""))
folded: dict[str, object] = {}
spellings: dict[str, list[str]] = {}
for reply_key, reply_val in scores.items():
folded_key = str(reply_key).strip().lower()
folded[folded_key] = reply_val
spellings.setdefault(folded_key, []).append(str(reply_key))
# Validate scores against rubric criteria
scale = rubric.get("scale", {"min": 1, "max": 5})
scale_min = scale.get("min", 1)
scale_max = scale.get("max", 5)
validated_scores: dict[str, float] = {}
for crit in rubric["criteria"]:
name = crit["name"]
key = str(name).strip().lower()
if key not in folded:
raise ValueError(f"Judge response has no score for criterion {name!r}")
if len(spellings[key]) > 1:
raise ValueError(
f"Judge response scores criterion {name!r} more than once, as "
+ ", ".join(repr(spelling) for spelling in spellings[key])
)
val = folded[key]
if isinstance(val, dict) and "score" in val:
val = val["score"]
if isinstance(val, bool):
val = None
try:
val = float(val)
except (TypeError, ValueError, OverflowError):
val = math.nan
if not math.isfinite(val):
raise ValueError(
f"Judge score for criterion {name!r} is not a number: {folded[key]!r}"
)
val = max(scale_min, min(scale_max, val))
validated_scores[name] = val
return validated_scores, reasoning
def _compute_weighted_score(scores: dict[str, float], rubric: dict) -> float:
"""Compute weighted average score from criteria scores."""
total_weight = 0.0
weighted_sum = 0.0
for crit in rubric["criteria"]:
weight = crit.get("weight", 1.0)
val = scores.get(crit["name"], 0.0)
weighted_sum += val * weight
total_weight += weight
return weighted_sum / total_weight if total_weight > 0 else 0.0
class JudgeEvaluator:
"""Configurable LLM-as-a-judge evaluator."""
# #1522: requests in a row whose retries were spent.
_spent_in_a_row = 0
def __init__(
self,
rubric: Optional[dict] = None,
provider: str = "openai",
model: str = "gpt-4o-mini",
api_base: Optional[str] = None,
api_key: Optional[str] = None,
):
if provider not in VALID_PROVIDERS:
raise ValueError(
f"Invalid provider '{provider}', "
f"must be one of: {', '.join(sorted(VALID_PROVIDERS))}"
)
validate_judge_api_base(api_base)
self.rubric = rubric or DEFAULT_RUBRIC
self.provider = provider
self.model = model
self.api_base = api_base
# Only use OpenAI API key for the openai provider to avoid leaking it
if provider == "openai":
self.api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
else:
self.api_key = api_key or ""
def evaluate(
self,
prompt: str,
response: str,
category: str = "default",
) -> JudgeScore:
"""Evaluate a single prompt-response pair."""
judge_prompt = _build_judge_prompt(prompt, response, self.rubric)
judge_response = self._call_llm(judge_prompt)
scores, reasoning = _parse_judge_response(judge_response, self.rubric)
weighted = _compute_weighted_score(scores, self.rubric)
return JudgeScore(
prompt=prompt,
response=response,
scores=scores,
weighted_score=weighted,
reasoning=reasoning,
category=category,
)
def evaluate_batch(
self,
items: list[dict],
) -> JudgeResults:
"""Evaluate a batch of prompt-response pairs.
Args:
items: List of dicts with 'prompt', 'response', optional 'category'.
Returns:
JudgeResults with aggregated scores.
"""
judge_scores: list[JudgeScore] = []
for item in items:
score = self.evaluate(
prompt=item["prompt"],
response=item["response"],
category=item.get("category", "default"),
)
judge_scores.append(score)
results = JudgeResults(scores=judge_scores)
results.compute()
return results
def compare_pair(self, prompt: str, resp_a: str, resp_b: str) -> int:
"""One pairwise A/B judgment -> 0 (A) / 1 (B) / -1 (tie / unparseable reply).
A judge that cannot be reached raises :class:`JudgeUnavailableError`
(#1447). A failed call is not a tie: scoring it as one turned a dead
judge into a measured 0.5 win-rate and a DON'T SHIP verdict.
"""
judge_prompt = _PAIRWISE_INSTRUCTIONS.format(
prompt=prompt, resp_a=resp_a, resp_b=resp_b
)
return _parse_pairwise(self._call_llm(judge_prompt))
def _call_llm(self, prompt: str, max_tokens: int = 1024) -> str:
"""Call the judge LLM. Uses OpenAI-compatible API for all providers.
``max_tokens`` caps the reply; GRPO's ``judge_rank`` reward sets it from
``training.grpo_judge_max_tokens`` (#2047). Every other caller keeps the
default, so its request is unchanged.
"""
if self.provider == "ollama":
base = self.api_base or "http://localhost:11434"
url = f"{base}/v1/chat/completions"
elif self.provider == "server":
base = self.api_base or "http://localhost:8000"
url = f"{base}/v1/chat/completions"
else:
base = self.api_base or "https://api.openai.com"
url = f"{base}/v1/chat/completions"
headers = {"Content-Type": "application/json"}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
payload = {
"model": self.model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.0,
"max_tokens": max_tokens,
}
try:
reply = _judge_request(url, payload, headers)
except JudgeUnavailableError as exc:
if not exc.retries_spent:
self._spent_in_a_row = 0 # a refusal or a bad reply is an answer
raise
self._spent_in_a_row += 1
if self._spent_in_a_row >= JUDGE_DOWN_AFTER:
raise JudgeDownError(
f"{self._spent_in_a_row} requests in a row failed, the last with "
f"{exc.detail}",
url=url,
retries_spent=True,
) from exc
raise
self._spent_in_a_row = 0
return reply
def _retry_delay(retry_after: Optional[str], attempt: int) -> float:
"""Seconds to pause before retry ``attempt`` (0-based), capped.
A numeric ``Retry-After`` wins; an HTTP-date one, or none, falls back to
exponential backoff.
"""
if retry_after:
try:
seconds = float(retry_after)
except ValueError:
seconds = math.nan
if math.isfinite(seconds): # nan and inf would reach time.sleep
return min(max(seconds, 0.0), JUDGE_MAX_BACKOFF_SECONDS)
return min(float(2 ** attempt), JUDGE_MAX_BACKOFF_SECONDS)
def _reply_content(resp, url: str) -> str:
"""The judge's text from an OpenAI-shaped reply, or ``JudgeUnavailableError``."""
try:
data = resp.json()
except ValueError as exc:
raise JudgeUnavailableError(f"reply is not JSON: {exc}", url=url) from exc
choices = data.get("choices") if isinstance(data, dict) else None
first = choices[0] if isinstance(choices, list) and choices else None
message = first.get("message") if isinstance(first, dict) else None
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, str):
raise JudgeUnavailableError("reply has no choices[0].message.content", url=url)
return content
def _judge_request(url: str, payload: dict, headers: dict, *, timeout: float = 120.0) -> str:
"""POST one judge request; retry 429 / 5xx / transport errors within a bound (#1447).
Every judge call goes through here, so every caller sees one failure shape:
``JudgeUnavailableError`` naming the URL, raised once the retries are spent,
at once for a 4xx that will not heal or an undecodable reply, and for a 200
with no usable content.
"""
import httpx
from soup_cli.utils.http_response import (
JUDGE_RESPONSE_LIMIT_BYTES,
ResponseTooLargeError,
post_capped,
)
failure = ""
for attempt in range(JUDGE_MAX_RETRIES + 1):
retry_after: Optional[str] = None
try:
resp = post_capped(
url, json=payload, headers=headers, timeout=timeout,
max_bytes=JUDGE_RESPONSE_LIMIT_BYTES,
endpoint_label="judge --api-base / --judge-api-base / judge_model",
)
except ResponseTooLargeError as exc:
raise JudgeUnavailableError(str(exc), url=url) from exc
except httpx.TransportError as exc:
failure = f"{type(exc).__name__}: {exc}"
except httpx.RequestError as exc: # e.g. DecodingError: not transient, no retry
raise JudgeUnavailableError(f"{type(exc).__name__}: {exc}", url=url) from exc
else:
if resp.status_code < 400:
return _reply_content(resp, url)
failure = f"HTTP {resp.status_code}"
if resp.status_code != 429 and resp.status_code < 500:
raise JudgeUnavailableError(failure, url=url)
retry_after = resp.headers.get("Retry-After")
if attempt < JUDGE_MAX_RETRIES:
delay = _retry_delay(retry_after, attempt)
logger.debug(
"judge request failed (%s); retry %d/%d in %.1fs",
failure, attempt + 1, JUDGE_MAX_RETRIES, delay,
)
_sleep(delay)
raise JudgeUnavailableError(
f"{failure} ({JUDGE_MAX_RETRIES + 1} attempts)", url=url, retries_spent=True
)
# ---------------------------------------------------------------------------
# Pairwise judging (v0.71.31) — shared by online-DPO + `soup ship --task-mode
# pairwise`. `compare_pair` (above) issues one A/B judgment; the free functions
# add swap-debiasing and a win-rate reduction.
# ---------------------------------------------------------------------------
_PAIRWISE_INSTRUCTIONS = (
"You are comparing two AI responses to the same prompt.\n\n"
"## Prompt\n{prompt}\n\n"
"## Response A\n{resp_a}\n\n"
"## Response B\n{resp_b}\n\n"
"## Task\nWhich response is better overall (helpfulness, accuracy, "
"safety)? Reply with a JSON object: {{\"winner\": \"A\"}} or "
"{{\"winner\": \"B\"}}. Return ONLY the JSON object."
)
def _parse_pairwise(text: str) -> int:
"""Parse a judge reply into 0 (A) / 1 (B) / -1 (tie or unparseable)."""
match = re.search(r'\{[^{}]*\}', text or "", re.DOTALL)
if match:
try:
data = json.loads(match.group())
winner = str(data.get("winner", "")).strip().upper()
if winner == "A":
return 0
if winner == "B":
return 1
except (json.JSONDecodeError, TypeError, ValueError):
pass
# Fallback: a bare "A" / "B" token.
stripped = (text or "").strip().upper()
if stripped.startswith("A") and not stripped.startswith("B"):
return 0
if stripped.startswith("B"):
return 1
return -1
def pairwise_compare(
prompt: str,
resp_a: str,
resp_b: str,
evaluator: "PairwiseJudge",
*,
swap: bool = True,
) -> int:
"""Return 0 (A preferred), 1 (B preferred), or -1 (tie / disagreement).
When ``swap`` is True the pair is judged in BOTH orders (A,B and B,A) and a
winner is returned ONLY if the two runs produce the same definite verdict —
the standard defence against a judge's positional bias. Anything else — a
tie or a disagreement — yields -1, so a single un-cross-checked verdict is
never trusted; a judge that cannot be reached raises instead (#1447). This
doubles the judge calls per pair vs a one-shot random-flip; the stronger
guarantee is intentional.
"""
first = evaluator.compare_pair(prompt, resp_a, resp_b)
if not swap:
return first
swapped = evaluator.compare_pair(prompt, resp_b, resp_a)
# Translate the swapped verdict back into A/B space: 0 -> B(1), 1 -> A(0).
if swapped == 0:
second = 1
elif swapped == 1:
second = 0
else:
second = -1
# Winner only when BOTH orders agree on a definite (0/1) verdict.
if first in (0, 1) and first == second:
return first
return -1
def pairwise_winrate(
pairs: "list[tuple[str, str, str]]", evaluator: "PairwiseJudge"
) -> float:
"""Tuned win-rate in [0, 1] over ``(prompt, base_resp, tuned_resp)`` triples.
Base is compared as A, tuned as B. A tuned win (verdict 1) scores 1.0, a tie
(-1) scores 0.5, a loss (0) scores 0.0. Empty input -> 0.5 (no evidence).
"""
if not pairs:
return 0.5
total = 0.0
for index, (prompt, base_resp, tuned_resp) in enumerate(pairs):
try:
verdict = pairwise_compare(prompt, base_resp, tuned_resp, evaluator, swap=True)
except JudgeUnavailableError as exc:
raise exc.for_rows(len(pairs) - index, len(pairs), "pairs") from exc
if verdict == 1:
total += 1.0
elif verdict == -1:
total += 0.5
return total / len(pairs)
def _as_prompt_text(prompt) -> str:
"""Best-effort text of a prompt (string or conversational message list)."""
if isinstance(prompt, str):
return prompt
if isinstance(prompt, list):
users = [
m["content"]
for m in prompt
if isinstance(m, dict) and m.get("role") == "user"
and isinstance(m.get("content"), str)
]
if users:
return users[-1]
return " ".join(
m["content"]
for m in prompt
if isinstance(m, dict) and isinstance(m.get("content"), str)
)
return str(prompt)
def _as_completion_text(completion) -> str:
"""Best-effort text of a completion (string, message, or message list)."""
if isinstance(completion, str):
return completion
if isinstance(completion, dict):
content = completion.get("content")
return content if isinstance(content, str) else str(completion)
if isinstance(completion, list):
return " ".join(
m["content"]
for m in completion
if isinstance(m, dict) and isinstance(m.get("content"), str)
)
return str(completion)
def make_judge_reward_func(evaluator: object, *, name: str = "soup_judge"):
"""Build a trl-1.x OnlineDPO POINTWISE reward function from a Soup evaluator.
trl 1.x removed the pairwise-judge Online DPO API (``BasePairwiseJudge``) and
ranks the two on-policy completions by a per-completion ``reward_funcs``
signal instead. This adapts Soup's ``JudgeEvaluator`` by scoring each
completion pointwise with ``evaluator.evaluate(prompt, completion)`` — the
SAME pointwise judge ``soup data best-of-n`` uses — returning its
``weighted_score``. Note the semantic difference vs the trl-0.19.x path,
which uses the swap-debiased *pairwise* comparison; per-version behaviour is
documented as a known difference.
The returned callable matches trl's reward-func contract
``fn(prompts, completions, **kwargs) -> list[float]`` and carries ``name`` as
``__name__`` so trl logs ``rewards/<name>``.
"""
def _reward(prompts, completions, **kwargs):
scores: list[float] = []
for prompt, completion in zip(prompts, completions, strict=False):
prompt_text = _as_prompt_text(prompt)
completion_text = _as_completion_text(completion)
try:
score = evaluator.evaluate(prompt_text, completion_text).weighted_score
scores.append(float(score))
except Exception as exc: # noqa: BLE001 — judge/network variety
logger.debug("judge reward func failed: %s", exc)
scores.append(0.0)
return scores
_reward.__name__ = name
return _reward
def _base_pairwise_judge_cls() -> type:
"""Lazily import TRL's ``BasePairwiseJudge`` with a friendly error."""
from soup_cli.trainer._trl_compat import resolve_trl_symbol
try:
return resolve_trl_symbol("BasePairwiseJudge", "trl.experimental.judges")
except ImportError as exc: # pragma: no cover — trl ships in [train]/[dev]
raise ImportError(
"SoupPairwiseJudge needs trl with BasePairwiseJudge support "
'(pip install "soup-cli[train]")'
) from exc
def make_soup_pairwise_judge(evaluator: "PairwiseJudge") -> "BasePairwiseJudge":
"""Build a TRL ``BasePairwiseJudge`` bound to a Soup ``JudgeEvaluator``.
Factory (not a module-level subclass) so ``eval/judge.py`` stays importable
without trl for the pure ``pairwise_*`` functions. ``judge`` returns, per
prompt, the index of the better completion (0 or 1), or ``-1`` when it
cannot name one: a tie, a failed or unparseable judge call, or a verdict
that changes when the two completions are swapped.
trl does NOT drop a ``-1`` (#1225). ``BasePairwiseJudge`` leaves invalid
ranks to the caller, and trl 0.29's ``OnlineDPOTrainer`` builds
``mask = rank == 0``, so a ``-1`` trains as if the second completion had
won. Soup's Online DPO trainer
(:func:`soup_cli.trainer.online_dpo_ranking.make_ranked_pairs_trainer`)
leaves those pairs out of the loss; any other caller must handle ``-1``
itself.
This is the one caller that keeps mapping a failed judge call to ``-1``
(#1447): the ranking trainer leaves ``-1`` pairs out of the loss, logs
``judge/invalid_rate`` and stops the run with an error naming the judge once
it has ranked nothing for too long (#1225), so a judge that is down ends the
run through that accounting, not a traceback.
"""
base_cls = _base_pairwise_judge_cls()
class _SoupPairwiseJudge(base_cls): # type: ignore[misc, valid-type]
def __init__(self, ev: "PairwiseJudge") -> None:
self.evaluator = ev
def judge(
self,
prompts: "list[str]",
completions: "list[list[str]]",
shuffle_order: bool = True,
) -> "list[int]":
out: list[int] = []
for prompt, pair in zip(prompts, completions, strict=False):
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
out.append(-1)
continue
try:
rank = pairwise_compare(
prompt, pair[0], pair[1], self.evaluator,
swap=shuffle_order,
)
except Exception as exc: # noqa: BLE001 — unranked; #1225 stops the run
logger.warning("online DPO judge call failed, pair left unranked: %s", exc)
rank = -1
out.append(rank)
return out
return _SoupPairwiseJudge(evaluator)