Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

2.9 KB
Raw Blame
"""Run the rendered validator command against actual dataset quality boundaries."""

import json
import shlex

import pytest
import yaml
from typer.testing import CliRunner

from soup_cli.cli import app
from soup_cli.utils.ci_workflow import render_soup_gate_workflow


def _validation_args(body):
    steps = yaml.safe_load(body)["jobs"]["soup-gate"]["steps"]
    step, = [item for item in steps if item.get("name") == "Validate training data"]
    command, = [line.strip() for line in step["run"].splitlines()
                if line.strip().startswith("soup data validate ")]
    args = shlex.split(command)
    assert args[:3] == ["soup", "data", "validate"]
    return args[1:]


@pytest.mark.parametrize("name", ["train.jsonl", "train's set.jsonl"])
@pytest.mark.parametrize("valid,total,expected", [(1, 10, 2), (9, 10, 2), (10, 10, 0)])
def test_rendered_gate_refuses_any_partial_drop(
    tmp_path, monkeypatch, name, valid, total, expected,
):
    monkeypatch.chdir(tmp_path)
    rows = [{"instruction": "hi", "output": "hello"}] * valid
    rows += [{"instruction": "missing output"}] * (total - valid)
    (tmp_path / name).write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
    body = render_soup_gate_workflow(
        data_path=name, suite_path="expectations.yaml", evidence_path="evidence.json",
    )
    runner = CliRunner()
    result = runner.invoke(app, _validation_args(body))
    assert result.exit_code == expected, (result.output, repr(result.exception))
    # The strict policy belongs to the generated gate, not standalone validation.
    standalone = runner.invoke(app, ["data", "validate", name])
    assert standalone.exit_code == 0, (standalone.output, repr(standalone.exception))


@pytest.mark.parametrize("content", ["", "\n", '{"instruction":"hi","output":"ok"}\n42\n'])
def test_rendered_gate_refuses_empty_or_skipped_rows(tmp_path, monkeypatch, content):
    monkeypatch.chdir(tmp_path)
    (tmp_path / "train.jsonl").write_text(content, encoding="utf-8")
    body = render_soup_gate_workflow(
        data_path="train.jsonl", suite_path="expectations.yaml", evidence_path="evidence.json",
    )
    result = CliRunner().invoke(app, _validation_args(body))
    assert result.exit_code in (2, 3), (result.output, repr(result.exception))


def test_ci_init_writes_strict_command_and_preserves_existing_workflow(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    runner = CliRunner()
    result = runner.invoke(app, ["ci", "init"])
    assert result.exit_code == 0, result.output
    workflow = tmp_path / ".github/workflows/soup-gate.yml"
    args = _validation_args(workflow.read_text(encoding="utf-8"))
    assert args[-2:] == ["--min-valid-fraction", "1.0"]
    workflow.write_text("operator-owned workflow\n", encoding="utf-8")
    result = runner.invoke(app, ["ci", "init"])
    assert result.exit_code == 1, result.output
    assert workflow.read_text(encoding="utf-8") == "operator-owned workflow\n"