Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Custom eval task runner — load JSONL eval sets, score model outputs."""
from __future__ import annotations
import json
import math
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional
from soup_cli.eval.generation import generate_outputs
from soup_cli.utils.final_answer import answers_match, parse_completion, parse_reference
from soup_cli.utils.safe_regex import check_config_regex
MAX_EVAL_TASKS = 10_000
MAX_REGEX_PATTERN_LEN = 1_000
MAX_REGEX_INPUT_LEN = 50_000
TASK_NAME = "custom"
VALID_SCORING = {
"exact", "contains", "answer", "regex", "semantic",
"tool_call_match", "tool_call_name_match", "tool_call_args_subset",
}
@dataclass
class EvalTask:
"""A single evaluation task loaded from JSONL."""
prompt: str
expected: str = ""
category: str = "default"
scoring: str = "exact"
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass
class EvalResult:
"""Result of evaluating a single task."""
task: EvalTask
output: str
score: float
matched: bool
@dataclass
class EvalResults:
"""Aggregated results from a full eval run."""
results: list[EvalResult]
total: int = 0
correct: int = 0
accuracy: float = 0.0
category_scores: dict[str, dict[str, float]] = field(default_factory=dict)
def compute(self) -> None:
"""Compute aggregate scores from individual results."""
self.total = len(self.results)
self.correct = sum(1 for r in self.results if r.matched)
self.accuracy = self.correct / self.total if self.total > 0 else 0.0
# Per-category breakdown
cats: dict[str, list[EvalResult]] = {}
for result in self.results:
cat = result.task.category
cats.setdefault(cat, []).append(result)
self.category_scores = {}
for cat, cat_results in sorted(cats.items()):
cat_correct = sum(1 for r in cat_results if r.matched)
cat_total = len(cat_results)
self.category_scores[cat] = {
"total": cat_total,
"correct": cat_correct,
"accuracy": cat_correct / cat_total if cat_total > 0 else 0.0,
}
def load_eval_tasks(path: "Path | str") -> list[EvalTask]:
"""Load eval tasks from a JSONL file with schema validation.
Each line must be a JSON object with at least a 'prompt' field.
Optional: 'expected', 'category', 'scoring'.
"""
path = Path(path)
if not path.exists():
raise FileNotFoundError(f"Eval tasks file not found: {path}")
if not path.suffix.lower() == ".jsonl":
raise ValueError(f"Expected .jsonl file, got: {path.suffix}")
tasks: list[EvalTask] = []
with open(path, encoding="utf-8") as fh:
for line_num, line in enumerate(fh, 1):
line = line.strip()
if not line:
continue
if len(tasks) >= MAX_EVAL_TASKS:
raise ValueError(
f"Eval file exceeds maximum of {MAX_EVAL_TASKS} tasks "
f"(stopped at line {line_num})"
)
try:
row = json.loads(line)
except json.JSONDecodeError as exc:
raise ValueError(
f"Invalid JSON on line {line_num}: {exc}"
) from exc
if not isinstance(row, dict):
raise ValueError(
f"Line {line_num}: expected JSON object, got {type(row).__name__}"
)
if "prompt" not in row:
raise ValueError(
f"Line {line_num}: missing required field 'prompt'"
)
scoring = row.get("scoring", "exact")
if not isinstance(scoring, str) or scoring not in VALID_SCORING:
raise ValueError(
f"Line {line_num}: invalid scoring '{scoring}', "
f"must be one of: {', '.join(sorted(VALID_SCORING))}"
)
expected = str(row.get("expected", ""))
if scoring == "answer":
require_single_answer(expected, f"Line {line_num}")
tasks.append(EvalTask(
prompt=str(row["prompt"]),
expected=expected,
category=str(row.get("category", "default")),
scoring=scoring,
metadata={
k: v for k, v in row.items()
if k not in ("prompt", "expected", "category", "scoring")
},
))
if not tasks:
raise ValueError(f"No eval tasks found in {path}")
return tasks
def require_single_answer(expected: str, where: str) -> None:
"""Refuse an ``expected`` that states no single answer for ``scoring: "answer"``."""
if parse_reference(expected) is None:
raise ValueError(
f"{where}: 'expected' {expected!r} states no single answer for scoring 'answer'; "
"give a bare answer, or end it with '#### <answer>' or \\boxed{<answer>}"
)
def score_exact(output: str, expected: str) -> bool:
"""Whole-output match (case-insensitive, stripped); ``The answer is 4.`` is not ``4``."""
return output.strip().lower() == expected.strip().lower()
def score_contains(output: str, expected: str) -> bool:
"""True when ``expected`` appears in ``output`` as a whole alnum-bounded token.
Case-insensitive via ``str.lower()`` on both sides. ``14`` does not contain ``4``,
and ``#### 420`` does not contain ``42``. A token match cannot read negation
(``not 42, so 41`` still contains ``42``); use ``scoring: "answer"`` to compare
the answer a model states.
"""
needle = expected.strip().lower()
if not needle:
return True
pattern = (
re.escape(needle)
+ rf"(?<{{{len(needle)}}})" # the character before the match
+ r"(?![A-Za-z0-9])" # the character after it
)
return re.search(pattern, output.lower()) is not None
def score_answer(output: str, expected: str) -> bool:
"""Compare the final answers both sides state, with GRPO's ``accuracy`` parser."""
reference = parse_reference(expected)
return reference is not None and answers_match(parse_completion(output), reference)
def score_regex(output: str, expected: str) -> bool:
"""Check if output matches expected regex pattern.
Guards against ReDoS: caps pattern and input length and rejects
structurally unsafe patterns before matching.
"""
if len(expected) > MAX_REGEX_PATTERN_LEN:
return False
truncated_output = output[:MAX_REGEX_INPUT_LEN]
try:
check_config_regex(expected, "eval.custom.expected")
return bool(re.search(expected, truncated_output, re.IGNORECASE))
except re.error:
return False
def score_semantic(output: str, expected: str) -> float:
"""Compute similarity via Jaccard token overlap (bag-of-words).
This is a lexical metric, not embedding-based semantic similarity.
Returns a float 0-1. Considered a match if similarity >= 0.5.
"""
out_tokens = set(output.strip().lower().split())
exp_tokens = set(expected.strip().lower().split())
if not out_tokens or not exp_tokens:
return 0.0
intersection = out_tokens & exp_tokens
union = out_tokens | exp_tokens
return len(intersection) / len(union)
def _parse_tool_call(text: str) -> Optional[dict]:
"""Parse a tool-call JSON blob. Returns None on any failure."""
if not isinstance(text, str):
return None
try:
data = json.loads(text)
except (json.JSONDecodeError, ValueError):
return None
if not isinstance(data, dict):
return None
return data
def _extract_function(call: dict) -> Optional[dict]:
func = call.get("function")
if not isinstance(func, dict):
return None
return func
def _parse_args(func: dict) -> Optional[dict]:
args = func.get("arguments")
if isinstance(args, dict):
return args
if isinstance(args, str):
try:
parsed = json.loads(args)
except (json.JSONDecodeError, ValueError):
return None
if isinstance(parsed, dict):
return parsed
return None
def _json_equal(expected: Any, actual: Any) -> bool:
"""Compare two parsed-JSON values the way JSON does, not the way Python does.
JSON has one boolean type and one number type; Python has ``bool`` as a subclass of
``int``, so ``True == 1`` and ``False == 0`` — including inside dicts and lists.
Comparing parsed tool-call arguments with ``==`` therefore gives a model that
answered ``1`` the full credit owed to one that answered ``true``, and a model that
answered ``false`` the credit for ``0``. Both are valid JSON, and they are different
arguments (#1755).
Numbers are deliberately left equal across forms: ``1 == 1.0`` still compares equal,
which is an existing control and not a policy change. A ``type(a) == type(b)`` guard
would separate them, which is why this rules on ``bool`` first and then defers to
``==``.
``expected`` here is the gold side of a user's eval task file, so both arguments are
user-supplied; neither is a Soup config field.
The walk is a worklist rather than recursion, so nesting depth costs heap and not
Python stack: ``json.loads`` accepts an argument hundreds of levels deep, and a
recursive comparison raised ``RecursionError`` on one instead of scoring it. The
pending list cannot cycle because both sides come from ``json.loads``, which cannot
build a self-referential document.
"""
pending: list[tuple[Any, Any]] = [(expected, actual)]
while pending:
want, got = pending.pop()
# bool before everything: it is a subclass of int, so every other branch below would
# otherwise treat True as 1.
if isinstance(want, bool) or isinstance(got, bool):
if not (isinstance(want, bool) and isinstance(got, bool) and want is got):
return False
continue
if isinstance(want, dict) and isinstance(got, dict):
if want.keys() != got.keys():
return False
pending.extend((want[key], got[key]) for key in want)
continue
if isinstance(want, list) and isinstance(got, list):
if len(want) != len(got):
return False
pending.extend(zip(want, got))
continue
# JSON has no NaN literal, but Python's parser accepts a bare NaN, so a task
# file can carry one. The old dict ``==`` matched NaN with NaN through CPython's
# shared singleton; keep that so recorded scores do not change (#1755).
if (
isinstance(want, float)
and isinstance(got, float)
and math.isnan(want)
and math.isnan(got)
):
continue
if want != got:
return False
return True
def tool_call_match(output: str, expected: str) -> bool:
"""Exact tool-call match: function name and arguments must match."""
out = _parse_tool_call(output)
exp = _parse_tool_call(expected)
if out is None or exp is None:
return False
out_func = _extract_function(out)
exp_func = _extract_function(exp)
if out_func is None or exp_func is None:
return False
if out_func.get("name") != exp_func.get("name"):
return False
out_args = _parse_args(out_func)
exp_args = _parse_args(exp_func)
if out_args is None or exp_args is None:
return False
return _json_equal(exp_args, out_args)
def tool_call_name_match(output: str, expected: str) -> bool:
"""Function-name-only match (ignores arguments)."""
out = _parse_tool_call(output)
exp = _parse_tool_call(expected)
if out is None or exp is None:
return False
out_func = _extract_function(out)
exp_func = _extract_function(exp)
if out_func is None or exp_func is None:
return False
name_out = out_func.get("name")
name_exp = exp_func.get("name")
return isinstance(name_out, str) and name_out == name_exp
def tool_call_args_subset(output: str, expected: str) -> float:
"""Partial-credit score for tool-call arguments (subset matching).
Returns 0.0–1.0:
- 0.5 weight for function-name match
- 0.5 weight for fraction of expected args present (with matching values)
"""
out = _parse_tool_call(output)
exp = _parse_tool_call(expected)
if out is None or exp is None:
return 0.0
out_func = _extract_function(out)
exp_func = _extract_function(exp)
if out_func is None or exp_func is None:
return 0.0
name_score = 0.5 if out_func.get("name") == exp_func.get("name") else 0.0
raw_out_args = _parse_args(out_func)
out_args = raw_out_args or {}
exp_args = _parse_args(exp_func) or {}
if raw_out_args is None and "arguments" in out_func:
args_score = 0.0
elif not exp_args:
# No args expected: full credit only if the model also produced none.
# Hallucinated args must NOT score full (the old `0.5 if ... else 0.5`
# dead ternary gave them full credit).
args_score = 0.5 if not out_args else 0.0
else:
matched = sum(
1
for k, v in exp_args.items()
if k in out_args and _json_equal(v, out_args[k])
)
args_score = 0.5 * (matched / len(exp_args))
return name_score + args_score
SCORING_FUNCTIONS = {
"exact": score_exact,
"contains": score_contains,
"answer": score_answer,
"regex": score_regex,
"tool_call_match": tool_call_match,
"tool_call_name_match": tool_call_name_match,
}
def score_task(task: EvalTask, output: str) -> EvalResult:
"""Score a single task output against the expected answer."""
if task.scoring == "semantic":
similarity = score_semantic(output, task.expected)
matched = similarity >= 0.5
return EvalResult(
task=task, output=output, score=similarity, matched=matched,
)
if task.scoring == "tool_call_args_subset":
similarity = tool_call_args_subset(output, task.expected)
matched = similarity >= 0.5
return EvalResult(
task=task, output=output, score=similarity, matched=matched,
)
scorer = SCORING_FUNCTIONS.get(task.scoring, score_exact)
matched = scorer(output, task.expected)
return EvalResult(
task=task, output=output, score=1.0 if matched else 0.0, matched=matched,
)
def run_eval(
model_path: str,
tasks: list[EvalTask],
generate_fn: Optional[object] = None,
) -> EvalResults:
"""Run evaluation on a list of tasks using a model.
Args:
model_path: Path to model directory.
tasks: List of EvalTask objects.
generate_fn: Optional callable(prompt: str) -> str, optionally exposing
batch(prompts) -> list[str]. If None, uses a transformers pipeline.
Returns:
EvalResults with per-task scores and aggregates.
"""
if generate_fn is None:
generate_fn = _create_default_generator(model_path)
results: list[EvalResult] = []
outputs = generate_outputs(generate_fn, [task.prompt for task in tasks])
for task, output in zip(tasks, outputs):
result = (
score_task(task, output) if output is not None
else EvalResult(task=task, output="", score=0.0, matched=False)
)
results.append(result)
eval_results = EvalResults(results=results)
eval_results.compute()
return eval_results
def _create_default_generator(model_path: str, trust_remote_code: bool = False):
"""Create a default text generation function using transformers pipeline."""
from rich.console import Console
from rich.panel import Panel
from transformers import pipeline
console = Console()
console.print(Panel(
f"[yellow]Loading model from: {model_path}[/]\n"
"This will execute model code from the specified path.",
title="Model Loading",
border_style="yellow",
))
pipe = pipeline(
"text-generation",
model=model_path,
max_new_tokens=512,
do_sample=False,
trust_remote_code=trust_remote_code,
)
def generate(prompt: str) -> str:
out = pipe(prompt, return_full_text=False)
return out[0]["generated_text"] if out else ""
return generate