Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

2.2 KB
Raw Blame
"""Keep parse failures distinct from empty tool arguments (#1734)."""

import json

import pytest

from soup_cli.eval.custom import EvalTask, score_task, tool_call_args_subset

BAD_ARGUMENTS = ["not-json", "[]", "null", [], None, 17]


def call(arguments, name="lookup"):
    return json.dumps({"function": {"name": name, "arguments": arguments}})


@pytest.mark.parametrize("arguments", BAD_ARGUMENTS)
@pytest.mark.parametrize("gold_arguments", [{}, {"query": "synthetic"}])
def test_provided_malformed_arguments_earn_no_argument_credit(arguments, gold_arguments):
    assert tool_call_args_subset(call(arguments), call(gold_arguments)) == 0.5


@pytest.mark.parametrize("arguments", [{}, "{}"])
def test_valid_empty_arguments_keep_full_credit(arguments):
    assert tool_call_args_subset(call(arguments), call({})) == 1.0


def test_extra_arguments_against_empty_gold_keep_only_name_credit():
    assert tool_call_args_subset(call({"extra": 1}), call({})) == 0.5


def test_valid_expected_subset_keeps_full_credit():
    assert tool_call_args_subset(
        call({"query": "synthetic", "limit": 2}), call({"query": "synthetic"}),
    ) == 1.0


@pytest.mark.parametrize("gold_arguments, score", [({}, 1.0), ({"query": "synthetic"}, 0.5)])
def test_omitted_arguments_keep_the_existing_treatment(gold_arguments, score):
    output = json.dumps({"function": {"name": "lookup"}})
    assert tool_call_args_subset(output, call(gold_arguments)) == score


def test_malformed_gold_handling_is_unchanged():
    assert tool_call_args_subset(call({}), call("not-json")) == 1.0


def test_score_task_keeps_the_half_credit_match_threshold():
    result = score_task(
        EvalTask(prompt="Synthetic question", expected=call({}), scoring="tool_call_args_subset"),
        call("not-json"),
    )
    assert result.score == 0.5
    assert result.matched is True  # The existing >= 0.5 threshold is a separate contract.


def test_invalid_arguments_cannot_add_credit_to_a_wrong_name():
    assert tool_call_args_subset(call("not-json", name="other"), call({})) == 0.0


def test_empty_arguments_with_wrong_name_keep_only_argument_credit():
    assert tool_call_args_subset(call({}, name="other"), call({})) == 0.5