Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

32.7 KB
Raw Blame
"""Reward functions for GRPO training.

Built-in reward functions:
  - accuracy: checks if the model answer matches the expected answer
  - format: checks if the response follows a structured format (e.g., <think>...</think>)
  - verifiable: RLVR — deterministic reward via math_verify / code_exec / json_schema

Custom reward functions can be loaded from a Python file with a
`reward_fn(completions, **kwargs)` callable.
"""

from __future__ import annotations

import contextlib
import importlib.util
import json
import math
import os
import re
import shutil as _shutil
import subprocess
import sys
import tempfile
import threading
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import partial, wraps
from pathlib import Path
from typing import Any

from rich.console import Console
from rich.panel import Panel

from soup_cli.utils import final_answer

console = Console()

MAX_CODE_OUTPUT_BYTES = 10_000
CODE_EXEC_TIMEOUT_SECONDS = 5
# Concurrency cap — at most this many code_exec subprocesses run in parallel
# per reward batch. Prevents fork-storms on large num_generations values.
CODE_EXEC_MAX_PARALLEL = 4
CODE_EXEC_MAX_MEMORY_BYTES = 512 * 1024 * 1024  # 512 MB per run

_SANDBOX_SEMAPHORE = threading.Semaphore(CODE_EXEC_MAX_PARALLEL)

_CODE_EXEC_WARNING_SHOWN = False

# Cached isolation strategy — recomputed on demand when tests reset to None
_ISOLATION_STRATEGY_CACHE: "str | None" = None

# macOS sandbox-exec profile: default-deny, allow narrow process needs, block
# network and writes outside /tmp. Defence-in-depth on top of RLIMIT + socket
# patch + ephemeral cwd. See sandbox-exec(1) and Apple's seatbelt SBPL.
MACOS_SANDBOX_PROFILE = (
    "(version 1)"
    "(deny default)"
    "(allow process-fork)"
    "(allow process-exec)"
    "(allow signal (target self))"
    "(allow file-read*)"
    '(allow file-write* (subpath "/tmp") (subpath "/private/tmp") (subpath "/var/folders"))'
    "(allow sysctl-read)"
    # Narrow mach-lookup allowlist — broad ``(allow mach-lookup)`` permits
    # DNS / NSURLSession via launchd-brokered Mach IPC and effectively
    # bypasses ``(deny network*)``. The names below are required for the
    # interpreter to boot (entitlement / system-services lookup) but do
    # NOT include ``com.apple.SystemConfiguration`` or ``com.apple.dnssd``.
    '(allow mach-lookup'
    ' (global-name "com.apple.SecurityServer")'
    ' (global-name "com.apple.system.notification_center")'
    ' (global-name "com.apple.system.opendirectoryd.libinfo"))'
    "(deny network*)"
)

# Python-level socket monkey-patch prepended before any sandboxed user code.
# Best-effort (bypassable via os.system / ctypes) — defence-in-depth on top of
# the RLIMIT + namespace / sandbox-exec isolation. Shared so the v0.71.18 #110
# agent-eval sandbox reuses the exact same network guard.
SANDBOX_NETWORK_GUARD = (
    "import socket\n"
    "def _blocked(*a, **k):\n"
    "    raise OSError('network disabled in sandbox')\n"
    "socket.socket = _blocked\n"
    "socket.create_connection = _blocked\n"
)


def _get_safe_sandbox_env() -> dict[str, str]:
    """Construct a minimal, secret-free environment for sandboxed processes."""
    safe_keys = {
        "PATH",
        "LANG",
        "LC_ALL",
        "TMPDIR",
        "TERM",
        "TZ",
        # Windows OS runtime essentials
        "SYSTEMROOT",
        "SYSTEMDRIVE",
        "WINDIR",
        "PATHEXT",
        "COMSPEC",
        "TEMP",
        "TMP",
        "USERPROFILE",
    }
    env = {k: v for k, v in os.environ.items() if k.upper() in safe_keys}
    if "PATH" not in env and sys.platform != "win32":
        env["PATH"] = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
    if "LANG" not in env and sys.platform != "win32":
        env["LANG"] = "C.UTF-8"
    if "LC_ALL" not in env and sys.platform != "win32":
        env["LC_ALL"] = "C.UTF-8"
    if "HOME" not in env and sys.platform != "win32":
        env["HOME"] = "/tmp"
    return env


def _compute_isolation_strategy() -> str:
    """Detect best-available OS-level sandbox isolation for code_exec_reward.

    Returns one of:
      - "namespaces" : Linux with `os.unshare` available (Python 3.12+) — we
        will best-effort `unshare(CLONE_NEWUSER|CLONE_NEWNET|CLONE_NEWPID)` in
        the child preexec_fn. Falls back at runtime if unprivileged user
        namespaces are disabled (EPERM/ENOSYS).
      - "sandbox-exec" : macOS with `sandbox-exec` binary on PATH — we wrap
        argv with `sandbox-exec -p <profile>`.
      - "best-effort" : everything else (Windows, restricted Linux). Existing
        RLIMIT + socket-patch + ephemeral-cwd guards still apply.

    The result is cached after first call. Tests reset
    ``_ISOLATION_STRATEGY_CACHE`` to None to re-probe.
    """
    if sys.platform == "linux" and hasattr(os, "unshare"):
        return "namespaces"
    if sys.platform == "darwin" and _shutil.which("sandbox-exec") is not None:
        return "sandbox-exec"
    return "best-effort"


def _get_isolation_strategy() -> str:
    """Cached wrapper for ``_compute_isolation_strategy``."""
    global _ISOLATION_STRATEGY_CACHE
    if _ISOLATION_STRATEGY_CACHE is None:
        _ISOLATION_STRATEGY_CACHE = _compute_isolation_strategy()
    return _ISOLATION_STRATEGY_CACHE


# Linux unshare flags — matches kernel uapi/linux/sched.h. Hard-coded so we
# don't depend on a runtime constant import.
_CLONE_NEWUSER = 0x10000000
_CLONE_NEWNET = 0x40000000
_CLONE_NEWPID = 0x20000000


def _try_unshare_namespaces(strict: bool = False) -> None:
    """Best-effort: unshare into new user/net/pid namespaces. Silent on failure.

    Called from the POSIX preexec_fn after RLIMITs are set. If the kernel
    rejects the unshare (unprivileged user namespaces disabled, common on
    hardened distros), we silently fall back to RLIMIT + socket patch alone.
    When ``strict=True``, failures raise PermissionError / OSError instead of falling back.
    """
    unshare = getattr(os, "unshare", None)
    if unshare is None:
        if strict:
            raise PermissionError("os.unshare not available")
        return
    try:
        unshare(_CLONE_NEWUSER | _CLONE_NEWNET | _CLONE_NEWPID)
    except (OSError, ValueError):
        # EPERM / ENOSYS / EINVAL — unprivileged unshare not allowed.
        if strict:
            raise
        # Continue with weaker isolation rather than failing the run.
        pass


def _show_code_exec_warning_once() -> None:
    """Display a one-time warning panel when code_exec_reward is first used."""
    global _CODE_EXEC_WARNING_SHOWN
    if _CODE_EXEC_WARNING_SHOWN:
        return
    _CODE_EXEC_WARNING_SHOWN = True
    console.print(
        Panel(
            "[bold yellow]RLVR code_exec_reward is a BEST-EFFORT sandbox.[/]\n\n"
            "Model-generated code runs in a subprocess with:\n"
            "  - 5s wall-clock timeout\n"
            "  - 512MB RLIMIT_AS on POSIX (Linux/macOS)\n"
            "  - Restricted temporary working directory\n"
            "  - A Python-level socket monkey-patch\n\n"
            "[bold]The socket patch can be bypassed[/] by generated code "
            "invoking os.system / subprocess / ctypes. Network isolation is "
            "NOT enforced. Do not enable code_exec_reward on hosts that "
            "hold secrets or run alongside trusted services. Prefer running "
            "training inside a container/VM with no network interface.",
            title="code_exec_reward — security notice",
            border_style="yellow",
        )
    )


def accuracy_reward(completions: list[list[dict]], **kwargs) -> list[float]:
    """Reward 1.0 when the completion's final answer matches the gold's, else 0.0.

    The completion and the gold are read by the same parser,
    :mod:`soup_cli.utils.final_answer`: the answer after the last '####', in the last
    \\boxed{}, or after 'answer is' / 'answer:'; an unmarked completion by its last line and
    its last number. A numeric gold is compared by value ('42' equals '#### 42.0'), any other
    gold as normalised text (case, whitespace and LaTeX wrappers ignored). There is no partial
    credit (#1226): the old 0.5 for "the gold appears somewhere in the completion" paid
    '#### 420' for a gold of 42.

    Args:
        completions: list of message lists, each containing a completion with 'content'.
        **kwargs: must contain 'answer' — the expected answer for each prompt.

    Returns:
        List of float rewards (1.0 for correct, 0.0 for incorrect).
    """
    answers = kwargs.get("answer", [])
    rewards = []
    for completion, expected in zip(completions, answers):
        content = completion[-1]["content"] if completion else ""
        reference = None if expected is None else final_answer.parse_reference(str(expected))
        if reference is None:
            rewards.append(0.0)
            continue
        predicted = final_answer.parse_completion(content)
        rewards.append(1.0 if final_answer.answers_match(predicted, reference) else 0.0)
    return rewards


def format_reward(completions: list[list[dict]], **kwargs) -> list[float]:
    """Reward based on whether the response follows a structured reasoning format.

    Checks for:
      - <think>...</think> block (chain-of-thought)
      - A final answer section after the thinking block

    Args:
        completions: list of message lists.
        **kwargs: unused.

    Returns:
        List of float rewards (0.0 to 1.0).
    """
    rewards = []
    for completion in completions:
        content = completion[-1]["content"] if completion else ""
        score = 0.0
        # Check for <think> block
        if re.search(r"<think>.*?</think>", content, re.DOTALL):
            score += 0.5
        # Check for content after </think>
        after_think = re.split(r"</think>", content)
        if len(after_think) > 1 and after_think[-1].strip():
            score += 0.5
        rewards.append(score)
    return rewards


def _extract_answer(text: str) -> str | None:
    """Return the final answer ``text`` states explicitly ('####', \\boxed{}, 'answer is').

    Kept as a thin alias: the parser lives in :mod:`soup_cli.utils.final_answer` (#1226).
    """
    return final_answer.extract_final_answer(text)


# --- RLVR: verifiable rewards (Part C of v0.25.0) ---


def math_verify_reward(
    completions: list[list[dict]],
    tolerance: float = 1e-4,
    **kwargs,
) -> list[float]:
    """RLVR math reward: compare the completion's final answer with the gold's.

    Both sides are read by :mod:`soup_cli.utils.final_answer`, so a GSM8K-shaped gold such as
    '6*7=42\\n#### 42' is the number 42 (#1226). A numeric gold scores 1.0 within ``tolerance``
    and 0.6 within max(100 x ``tolerance``, 1e-2). A gold that is not a number, such as
    '\\frac{14}{3}' or 'p - q', is compared as normalised text: 1.0 on an exact match, else
    0.0. Security: never uses ``eval()``; nothing is evaluated.
    """
    answers = kwargs.get("answer", [])
    rewards: list[float] = []
    for completion, expected in zip(completions, answers):
        content = completion[-1]["content"] if completion else ""
        reference = None if expected is None else final_answer.parse_reference(str(expected))
        parsed = final_answer.parse_completion(content)
        if reference is None or parsed is None:
            rewards.append(0.0)
            continue
        if reference.number is None:
            matched = final_answer.answers_match(parsed, reference)
            rewards.append(1.0 if matched else 0.0)
            continue
        if parsed.number is None or final_answer.variables_conflict(parsed, reference):
            rewards.append(0.0)
            continue
        try:
            error = float(abs(parsed.number - reference.number))
        except ArithmeticError:  # an exponent beyond Decimal's range cannot be compared
            rewards.append(0.0)
            continue

        if error <= tolerance:
            rewards.append(1.0)
        elif error <= max(tolerance * 100, 1e-2):
            rewards.append(0.6)
        else:
            rewards.append(0.0)
    return rewards


def _extract_code_block(content: str) -> str:
    """Extract a Python code block from content. Strips markdown fences."""
    match = re.search(r"```(?:python)?\s*(.*?)```", content, re.DOTALL)
    if match:
        return match.group(1).strip()
    return content.strip()


def _apply_rlimit(strict_namespaces: bool = False) -> None:
    """POSIX only: set resource limits for sandboxed subprocess.

    Called via ``preexec_fn`` before the child runs user code. On Windows this
    is never invoked because ``preexec_fn`` is POSIX-only and the caller skips
    it there.
    """
    try:
        import resource

        resource.setrlimit(
            resource.RLIMIT_AS,
            (CODE_EXEC_MAX_MEMORY_BYTES, CODE_EXEC_MAX_MEMORY_BYTES),
        )
        resource.setrlimit(
            resource.RLIMIT_CPU,
            (CODE_EXEC_TIMEOUT_SECONDS, CODE_EXEC_TIMEOUT_SECONDS),
        )
        if hasattr(resource, "RLIMIT_FSIZE"):
            # Limit file size created by subprocess (10 MB max)
            resource.setrlimit(
                resource.RLIMIT_FSIZE, (10 * 1024 * 1024, 10 * 1024 * 1024)
            )
        if hasattr(resource, "RLIMIT_CORE"):
            # Disable core dumps
            resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
        if hasattr(resource, "RLIMIT_NOFILE"):
            # Cap open file descriptors
            resource.setrlimit(resource.RLIMIT_NOFILE, (256, 256))
        if hasattr(resource, "RLIMIT_NPROC"):
            # Cap child processes / threads
            resource.setrlimit(resource.RLIMIT_NPROC, (64, 64))
    except (ImportError, ValueError, OSError):
        pass
    # Linux defence-in-depth: best-effort unshare into private namespaces.
    if _get_isolation_strategy() == "namespaces":
        _try_unshare_namespaces(strict=strict_namespaces)


@dataclass
class SandboxProcessResult:
    returncode: int | None
    stdout: str
    stderr: str
    timed_out: bool = False
    output_exceeded: bool = False
    launch_failed: bool = False


class SandboxIsolationError(PermissionError):
    """A strict sandbox run could not start its isolated child.

    Raised by ``_run_code_sandbox(..., strict_namespaces=True)`` when the child
    fails to launch because the Linux namespaces could not be set up in the
    pre-exec hook. The code did not run. (A ``sandbox-exec`` wrapper that fails
    on macOS is a non-zero exit of a launched child, not this error.)
    """


def _run_sandboxed_subprocess(
    argv: list[str],
    preexec_fn: Callable[[], None] | None = None,
    max_output_bytes: int = MAX_CODE_OUTPUT_BYTES,
    timeout_seconds: int = CODE_EXEC_TIMEOUT_SECONDS,
    env: dict[str, str] | None = None,
) -> SandboxProcessResult:
    """Execute a sandboxed subprocess under concurrency limit with streaming kill-on-overflow."""
    if _get_isolation_strategy() == "sandbox-exec" and sys.platform == "darwin":
        sandbox_bin = _shutil.which("sandbox-exec") or "/usr/bin/sandbox-exec"
        argv = [sandbox_bin, "-p", MACOS_SANDBOX_PROFILE, *argv]

    if env is None:
        env = _get_safe_sandbox_env()

    with _SANDBOX_SEMAPHORE, tempfile.TemporaryDirectory(prefix="soup-sandbox-") as tmpdir:
        try:
            proc = subprocess.Popen(
                argv,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                cwd=tmpdir,
                preexec_fn=preexec_fn,
                env=env,
            )
        except (PermissionError, OSError, subprocess.SubprocessError) as exc:
            return SandboxProcessResult(
                returncode=None,
                stdout="",
                stderr=str(exc),
                launch_failed=True,
            )

        stdout_chunks: list[bytes] = []
        stderr_chunks: list[bytes] = []
        output_exceeded = False
        lock = threading.Lock()

        def _reader(stream, chunks: list[bytes]) -> None:
            nonlocal output_exceeded
            try:
                while True:
                    chunk = stream.read(4096)
                    if not chunk:
                        break
                    with lock:
                        chunks.append(chunk)
                        total = sum(len(c) for c in stdout_chunks) + sum(
                            len(c) for c in stderr_chunks
                        )
                        if total > max_output_bytes:
                            output_exceeded = True
                            with contextlib.suppress(Exception):
                                proc.kill()
                            break
            finally:
                stream.close()

        t_out = threading.Thread(target=_reader, args=(proc.stdout, stdout_chunks))
        t_err = threading.Thread(target=_reader, args=(proc.stderr, stderr_chunks))
        t_out.daemon = True
        t_err.daemon = True
        t_out.start()
        t_err.start()

        try:
            proc.wait(timeout=timeout_seconds)
            timed_out = False
        except subprocess.TimeoutExpired:
            timed_out = True
            with contextlib.suppress(Exception):
                proc.kill()
            with contextlib.suppress(Exception):
                proc.wait(timeout=1.0)

        t_out.join(timeout=1.0)
        t_err.join(timeout=1.0)

        if timed_out:
            return SandboxProcessResult(None, "", "", timed_out=True)

        if output_exceeded:
            return SandboxProcessResult(
                proc.returncode if proc.returncode is not None else 1,
                "",
                "sandbox output exceeded limit",
                output_exceeded=True,
            )

        raw_out = b"".join(stdout_chunks)
        raw_err = b"".join(stderr_chunks)
        if len(raw_out) + len(raw_err) > max_output_bytes:
            return SandboxProcessResult(
                proc.returncode if proc.returncode is not None else 1,
                "",
                "sandbox output exceeded limit",
                output_exceeded=True,
            )

        stdout = raw_out.decode("utf-8", errors="replace").strip()
        stderr = raw_err.decode("utf-8", errors="replace").strip()
        return SandboxProcessResult(proc.returncode, stdout, stderr)


def _run_code_sandbox(code: str, strict_namespaces: bool = False) -> "str | None":
    """Run code in a subprocess sandbox with timeout, rlimits, and output caps.

    Security posture (best-effort, NOT a strong sandbox):
    - Hard wall-clock timeout via subprocess.
    - POSIX ``RLIMIT_AS`` (address space) and ``RLIMIT_CPU`` via preexec_fn.
    - Output truncated to ``MAX_CODE_OUTPUT_BYTES``.
    - Python-level socket monkey-patch (bypassable via os.system / ctypes).
    - Subprocess cwd is a freshly created temporary directory per run, so
      the child's default relative writes land in an ephemeral sandbox dir.
    - Uses ``python -I -S`` to disable site packages and user customization.

    ``strict_namespaces`` (default off) is for ``soup serve``'s
    ``/v1/tools/python``: as in ``_run_bash_sandbox``, a failed namespace
    ``unshare`` under the ``namespaces`` strategy stops the launch instead of
    falling back, and a launch failure raises :class:`SandboxIsolationError`
    instead of returning None. The reward and recipe callers keep the default,
    which runs the code without the namespaces where they cannot be created.

    Returns stdout string or None on failure.
    """
    _show_code_exec_warning_once()

    wrapped = SANDBOX_NETWORK_GUARD + "\n" + code

    preexec: Callable[[], None] | None = None
    if sys.platform != "win32":
        preexec = (
            partial(_apply_rlimit, strict_namespaces=True)
            if strict_namespaces
            else _apply_rlimit
        )

    argv: list[str] = [sys.executable, "-I", "-S", "-c", wrapped]

    result = _run_sandboxed_subprocess(argv, preexec)
    if strict_namespaces and result.launch_failed:
        raise SandboxIsolationError(result.stderr)
    if result.returncode != 0 or result.timed_out or result.output_exceeded or result.launch_failed:
        return None
    return result.stdout


def _run_bash_sandbox(command: str) -> SandboxProcessResult:
    """Run bash command in a subprocess sandbox with timeout, rlimits, and output caps.

    Security posture: mirrors _run_code_sandbox, but does not use Python-level
    socket monkey-patching. Relies entirely on OS-level isolation (unshare on Linux,
    sandbox-exec on macOS).
    Not supported on Windows.
    """
    if sys.platform == "win32":
        raise NotImplementedError("bash sandbox not supported on Windows")

    _show_code_exec_warning_once()

    def preexec() -> None:
        _apply_rlimit(strict_namespaces=True)

    bash_bin = _shutil.which("bash") or "/bin/bash"
    argv: list[str] = [bash_bin, "-c", command]

    result = _run_sandboxed_subprocess(argv, preexec)
    if result.launch_failed:
        raise PermissionError(result.stderr)
    return result


def code_exec_reward(
    completions: list[list[dict]],
    **kwargs,
) -> list[float]:
    """RLVR code reward: execute completion code, compare output to expected.

    Security: runs every completion in a subprocess sandbox with a 5s timeout
    and a 10KB output cap. Network access is disabled via socket monkey-patch
    injected before user code runs. Per-batch parallelism is capped at
    ``CODE_EXEC_MAX_PARALLEL`` to prevent fork storms on large batches.
    """
    from concurrent.futures import ThreadPoolExecutor

    expected_outputs = kwargs.get("expected", kwargs.get("answer", []))
    items: list[tuple[str, str]] = []
    for completion, expected in zip(completions, expected_outputs):
        content = completion[-1]["content"] if completion else ""
        code = _extract_code_block(content)
        items.append((code, str(expected).strip()))

    def _score(item: tuple[str, str]) -> float:
        code, expected = item
        if not code:
            return 0.0
        output = _run_code_sandbox(code)
        if output is None:
            return 0.0
        return 1.0 if output.strip() == expected else 0.0

    max_workers = max(1, min(CODE_EXEC_MAX_PARALLEL, len(items)))
    if not items:
        return []
    with ThreadPoolExecutor(max_workers=max_workers) as pool:
        return list(pool.map(_score, items))


def _score_against_schema(data: object, schema: dict) -> float:
    """Lightweight JSON-schema completeness score (no jsonschema dep).

    - 1.0 if all required fields are present with matching primitive types.
    - Partial credit proportional to required fields satisfied.
    - 0.0 if schema validation fails completely.
    """
    if not isinstance(schema, dict):
        return 0.0
    if schema.get("type") == "object":
        if not isinstance(data, dict):
            return 0.0
        required = schema.get("required") or list((schema.get("properties") or {}).keys())
        if not required:
            return 1.0
        hit = 0
        properties = schema.get("properties") or {}
        for field_name in required:
            if field_name not in data:
                continue
            field_schema = properties.get(field_name, {})
            if _type_matches(data[field_name], field_schema.get("type")):
                hit += 1
        return hit / len(required)
    return 1.0 if _type_matches(data, schema.get("type")) else 0.0


def _type_matches(value: object, type_name: "str | None") -> bool:
    if type_name is None:
        return True
    mapping = {
        "string": str,
        "integer": int,
        "number": (int, float),
        "boolean": bool,
        "array": list,
        "object": dict,
        "null": type(None),
    }
    expected_type = mapping.get(type_name)
    if expected_type is None:
        return True
    if type_name == "integer" and isinstance(value, bool):
        return False
    return isinstance(value, expected_type)


def json_schema_reward(
    completions: list[list[dict]],
    **kwargs,
) -> list[float]:
    """RLVR JSON schema reward: parse completion as JSON, score schema conformance."""
    schemas = kwargs.get("schema", [])
    rewards: list[float] = []
    for completion, schema in zip(completions, schemas):
        if isinstance(schema, str):
            try:
                schema = json.loads(schema)
            except (json.JSONDecodeError, ValueError):
                rewards.append(0.0)
                continue
        content = completion[-1]["content"] if completion else ""
        # Strip markdown fences first
        fenced = re.search(r"```(?:json)?\s*(.*?)```", content, re.DOTALL)
        if fenced:
            content = fenced.group(1)
        try:
            data = json.loads(content.strip())
        except (json.JSONDecodeError, ValueError):
            rewards.append(0.0)
            continue
        rewards.append(_score_against_schema(data, schema))
    return rewards


# Registry of built-in reward functions
BUILTIN_REWARDS: dict[str, Callable] = {
    "accuracy": accuracy_reward,
    "format": format_reward,
}

VERIFIABLE_DOMAINS: dict[str, Callable] = {
    "math": math_verify_reward,
    "code": code_exec_reward,
    "json_schema": json_schema_reward,
}


#: A reward function that carries this attribute set to True may score a
#: completion ``None``: no score, which TRL 0.29 turns into NaN and leaves out of
#: the logged mean (#2047). Every other reward must return finite numbers.
REWARD_MAY_RETURN_NONE = "_soup_reward_may_return_none"


def _validated_reward_fn(reward_fn: Callable) -> Callable:
    """Wrap one reward function with TRL's one-score-per-completion contract.

    ``None`` passes only for a reward that declares :data:`REWARD_MAY_RETURN_NONE`.
    """
    reward_name = getattr(reward_fn, "__name__", "reward_fn")
    allows_none = getattr(reward_fn, REWARD_MAY_RETURN_NONE, False) is True

    @wraps(reward_fn)
    def checked(*args: Any, **kwargs: Any) -> Any:
        rewards = reward_fn(*args, **kwargs)
        completions = kwargs.get("completions")
        if completions is None:
            if len(args) >= 2:
                completions = args[1]
            elif args:
                completions = args[0]
        if completions is None:
            raise ValueError(
                f"Reward function {reward_name!r} was called without completions"
            )
        try:
            reward_count = len(rewards)
        except TypeError as exc:
            raise ValueError(
                f"Reward function {reward_name!r} must return one finite score "
                "per completion"
            ) from exc
        completion_count = len(completions)
        if reward_count != completion_count:
            raise ValueError(
                f"Reward function {reward_name!r} returned {reward_count} scores "
                f"for {completion_count} completions; return exactly one finite "
                "score per completion and verify the required dataset columns"
            )
        for index, reward in enumerate(rewards):
            if reward is None and allows_none:
                continue
            if isinstance(reward, bool):
                raise ValueError(
                    f"Reward function {reward_name!r} returned boolean score at "
                    f"index {index}; scores must be finite numbers"
                )
            try:
                finite = math.isfinite(float(reward))
            except (TypeError, ValueError, OverflowError):
                finite = False
            if not finite:
                raise ValueError(
                    f"Reward function {reward_name!r} returned non-finite or "
                    f"non-numeric score {reward!r} at index {index}"
                )
        return rewards

    return checked


def validate_reward_funcs(reward_funcs: Any) -> Any:
    """Validate one reward callable or an ensemble without changing its shape."""
    if isinstance(reward_funcs, (list, tuple)):
        return [_validated_reward_fn(reward_fn) for reward_fn in reward_funcs]
    return _validated_reward_fn(reward_funcs)


def load_reward_fn(
    reward_fn_spec: str, verifiable_domain: "str | None" = None,
) -> Callable:
    """Load a reward function by name or from a custom Python file.

    Args:
        reward_fn_spec: Either a built-in name ('accuracy', 'format', 'verifiable')
            or a path to a .py file containing a `reward_fn` callable.
        verifiable_domain: Required when ``reward_fn_spec == 'verifiable'``.
            One of ``"math"``, ``"code"``, ``"json_schema"``.

    Returns:
        A callable reward function with signature:
        (completions: list[list[dict]], **kwargs) -> list[float]
    """
    # RLVR: verifiable reward routing
    if reward_fn_spec == "verifiable":
        if verifiable_domain is None:
            raise ValueError(
                "reward_fn='verifiable' requires verifiable_domain "
                "(one of: math, code, json_schema)"
            )
        if verifiable_domain not in VERIFIABLE_DOMAINS:
            raise ValueError(
                f"Unknown verifiable_domain: '{verifiable_domain}'. "
                f"Options: {', '.join(VERIFIABLE_DOMAINS.keys())}"
            )
        console.print(
            f"[dim]Using verifiable reward: domain={verifiable_domain}[/]"
        )
        return VERIFIABLE_DOMAINS[verifiable_domain]

    if reward_fn_spec == "judge_rank":
        raise ValueError(
            "judge_rank needs a GRPO config: it is built from training.grpo_judge and "
            "the other training.grpo_judge_* fields, so it runs only as a task: grpo "
            "reward (#2047)"
        )

    # Built-in reward function
    if reward_fn_spec in BUILTIN_REWARDS:
        console.print(f"[dim]Using built-in reward function: {reward_fn_spec}[/]")
        return BUILTIN_REWARDS[reward_fn_spec]

    # Custom Python file
    reward_path = Path(reward_fn_spec)
    if reward_path.exists() and reward_path.suffix == ".py":
        console.print(
            f"[bold yellow]Warning:[/] Loading custom reward function from: "
            f"[bold]{reward_path.resolve()}[/]\n"
            f"[yellow]This will execute arbitrary Python code. "
            f"Only use reward files you trust.[/]"
        )
        spec = importlib.util.spec_from_file_location("custom_reward", reward_path)
        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)

        if not hasattr(module, "reward_fn"):
            raise ValueError(
                f"Custom reward file {reward_path} must define a 'reward_fn' callable.\n"
                f"Example:\n"
                f"  def reward_fn(completions, **kwargs):\n"
                f"      return [1.0] * len(completions)"
            )
        return module.reward_fn

    raise ValueError(
        f"Unknown reward function: '{reward_fn_spec}'\n"
        f"Options: {', '.join(BUILTIN_REWARDS.keys())}, 'verifiable', "
        f"or path to a .py file"
    )


def load_reward_fns(
    reward_fn_spec: str,
    verifiable_domain: "str | None" = None,
    *,
    builders: "Mapping[str, Callable[[], Callable]] | None" = None,
) -> list[Callable]:
    """Load one OR MORE reward functions from a comma-separated spec (v0.71.40 #311).

    ``reward_fn`` may name several rewards, e.g. ``"accuracy,format"`` — each is
    resolved via :func:`load_reward_fn` and returned as a list, which TRL's
    ``GRPOTrainer(reward_funcs=[...])`` accepts and which the ``rm_ensemble``
    reward-hack detector (needs >= 2 reward fns) unlocks. A single name still
    returns a one-element list, so callers can always treat the result uniformly.

    Splitting is on ``,``; blank / empty segments raise (``"accuracy,"`` is a typo,
    not "accuracy plus nothing"), and duplicate segments raise (they would collide
    by ``__name__`` in the ``rm_ensemble`` capture buffer, silently shrinking the
    ensemble). A ``.py`` path containing a literal comma is not supported — rename
    the file.

    ``builders`` maps a segment name to a function that builds that reward from
    the caller's config (GRPO's ``judge_rank``, #2047); every other segment goes
    through :func:`load_reward_fn`.
    """
    if not isinstance(reward_fn_spec, str):
        raise ValueError(
            "reward_fn must be a string (a name, a .py path, or a comma-separated "
            f"list), got {type(reward_fn_spec).__name__}"
        )
    if not reward_fn_spec.strip():
        raise ValueError("reward_fn must not be blank")
    segments = [seg.strip() for seg in reward_fn_spec.split(",")]
    if any(not seg for seg in segments):
        raise ValueError(
            f"reward_fn {reward_fn_spec!r} has an empty comma segment — "
            "remove the stray comma"
        )
    seen: set[str] = set()
    for seg in segments:
        if seg in seen:
            raise ValueError(
                f"reward_fn {reward_fn_spec!r} lists {seg!r} twice — "
                "duplicate rewards collide in the ensemble"
            )
        seen.add(seg)
    builders = builders or {}
    return [
        builders[seg]() if seg in builders
        else load_reward_fn(seg, verifiable_domain=verifiable_domain)
        for seg in segments
    ]