Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

22.3 KB
Raw Blame
"""Unit tests for Automated Dataset Cleaning & Sanity Repair Pipeline."""

from __future__ import annotations

import hashlib
import json
import re
from pathlib import Path

import pytest
from typer.testing import CliRunner

from soup_cli.cli import app
from soup_cli.utils.data_clean import (
    CleanReport,
    clean_dataset,
    clean_row,
    is_echo_turn,
    repair_code_fences,
    repair_json_string,
    sanitize_text,
    strip_boilerplate,
)

runner = CliRunner()

_ANSI_RE = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]")


def _strip_ansi(text: str) -> str:
    """Strip ANSI escape sequences and collapse whitespace."""
    no_ansi = _ANSI_RE.sub("", text)
    return re.sub(r"\s+", " ", no_ansi).strip()


def _hash_file(path: Path) -> str:
    """Compute SHA-256 checksum of a file."""
    hasher = hashlib.sha256()
    with open(path, "rb") as f:
        while chunk := f.read(8192):
            hasher.update(chunk)
    return hasher.hexdigest()


def _create_sample_dirty_file(directory: Path) -> Path:
    """Create a temporary dirty JSONL file for CLI testing."""
    file_path = directory / "dirty_data.jsonl"
    data = [
        {
            "messages": [
                {"role": "user", "content": "How to add in Python?\u200b"},
                {
                    "role": "assistant",
                    "content": (
                        "Certainly! As an AI language model, here is the function:\n\n"
                        "```python\ndef add(a, b):\n    return a + b\n"
                    ),
                },
            ]
        },
        {
            "messages": [
                {"role": "user", "content": "What is Python?"},
                {"role": "assistant", "content": "Python is a programming language."},
            ]
        },
        {
            "messages": [
                {"role": "user", "content": "Empty question"},
                {"role": "assistant", "content": "   \n\u200b   "},
            ]
        },
    ]
    with open(file_path, "w", encoding="utf-8") as file_handle:
        for row in data:
            file_handle.write(json.dumps(row) + "\n")
    return file_path


def test_sanitize_text_control_characters():
    """Verify zero-width spaces, C0 controls, and CRLF are properly cleaned."""
    dirty_text = "Hello\u200b world\x00\x07!\r\nSecond line."
    cleaned, modified = sanitize_text(dirty_text)
    assert modified is True
    assert cleaned == "Hello world!\nSecond line."


def test_sanitize_text_clean_noop():
    """Verify clean text returns modified=False."""
    clean_text = "Standard clean text with\nnewlines and tabs\there."
    cleaned, modified = sanitize_text(clean_text)
    assert modified is False
    assert cleaned == clean_text


def test_repair_code_fences_unclosed():
    """Verify unclosed markdown code blocks are safely closed."""
    unclosed = "Here is the code:\n```python\ndef add(a, b):\n    return a + b\n"
    cleaned, modified = repair_code_fences(unclosed)
    assert modified is True
    assert cleaned.endswith("\n```")
    assert cleaned.count("```") == 2


def test_repair_code_fences_already_balanced():
    """Verify balanced code blocks are left intact."""
    balanced = "Here is the code:\n```python\ndef add(a, b):\n    return a + b\n```"
    cleaned, modified = repair_code_fences(balanced)
    assert modified is False
    assert cleaned == balanced


def test_strip_boilerplate_prefixes():
    """Verify common LLM preamble is stripped."""
    text = "Certainly! As an AI language model, I can explain that. Quantum computing uses qubits."
    cleaned, modified = strip_boilerplate(text)
    assert modified is True
    assert cleaned == "Quantum computing uses qubits."


def test_strip_boilerplate_suffixes():
    """Verify common LLM sign-offs are stripped."""
    text = "Here is the answer. I hope this helps! Please let me know if you have any questions."
    cleaned, modified = strip_boilerplate(text)
    assert modified is True
    assert cleaned == "Here is the answer."


def test_repair_json_string_trailing_comma():
    """Verify trailing commas in JSON are repaired."""
    invalid_json = '{"name": "test", "value": 42,}'
    repaired, modified = repair_json_string(invalid_json)
    assert modified is True
    parsed = json.loads(repaired)
    assert parsed["name"] == "test"
    assert parsed["value"] == 42


def test_repair_json_string_markdown_wrapped():
    """Verify markdown-wrapped JSON is extracted cleanly."""
    wrapped_json = '```json\n{"action": "search", "query": "orders"}\n```'
    repaired, modified = repair_json_string(wrapped_json)
    assert modified is True
    parsed = json.loads(repaired)
    assert parsed["action"] == "search"
    assert parsed["query"] == "orders"


def test_is_echo_turn():
    """Verify echo turns where assistant repeats prompt verbatim are flagged."""
    assert is_echo_turn("What is your return policy?", "what is your return policy?") is True
    assert is_echo_turn("What is your return policy?", "Our return policy is 30 days.") is False


def test_clean_row_chatml_with_heuristics():
    """Test cleaning a full ChatML row when opt-in heuristics are enabled."""
    row = {
        "messages": [
            {"role": "user", "content": "How do I add in Python?\u200b"},
            {
                "role": "assistant",
                "content": (
                    "Sure! As an AI language model, here is the function:\n\n"
                    "```python\ndef add(a, b):\n    return a + b\n"
                ),
            },
        ]
    }
    cleaned, rules = clean_row(
        row,
        "chatml",
        strip_ai_boilerplate=True,
        repair_code=True,
    )
    assert cleaned is not None
    assert "Invisible & Control Chars" in rules
    assert "Boilerplate Disclaimers" in rules
    assert "Markdown Code Fence Repair" in rules

    assistant_msg = cleaned["messages"][1]["content"]
    assert "As an AI" not in assistant_msg
    assert assistant_msg.endswith("```")


def test_clean_row_preserves_unknown_keys_and_roles():
    """B1: Verify clean_row preserves unknown keys ('train', 'name') and roles ('developer')."""
    row = {
        "custom_id": "row_12345",
        "metadata": {"source": "synthetic"},
        "messages": [
            {"role": "developer", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Hello", "name": "bob"},
            {"role": "assistant", "content": "Hi Bob!", "train": False},
        ],
    }
    cleaned, rules = clean_row(row, "chatml")
    assert cleaned is not None
    assert len(rules) == 0  # 100% clean row

    # Assert row-level keys are strictly preserved
    assert cleaned["custom_id"] == "row_12345"
    assert cleaned["metadata"] == {"source": "synthetic"}

    # Assert message-level keys and roles are strictly preserved
    assert cleaned["messages"][0]["role"] == "developer"
    assert cleaned["messages"][1]["name"] == "bob"
    assert cleaned["messages"][2]["train"] is False

    # Byte-identical verification
    assert json.dumps(cleaned) == json.dumps(row)


def test_clean_row_alpaca():
    """Test cleaning an Alpaca row with opt-in boilerplate stripping."""
    row = {
        "instruction": "Explain gravity\u200b",
        "input": "",
        "output": "Certainly! Gravity is a fundamental force. I hope this helps!",
    }
    cleaned, rules = clean_row(row, "alpaca", strip_ai_boilerplate=True)
    assert cleaned is not None
    assert "Boilerplate Disclaimers" in rules
    assert cleaned["output"] == "Gravity is a fundamental force."


def test_clean_row_alpaca_no_input_key_no_synthesis():
    """Verify alpaca row without 'input' key does not synthesize 'input' key."""
    row = {"instruction": "Say hi", "output": "hi", "custom": "keep"}
    cleaned, rules = clean_row(row, "alpaca")
    assert cleaned is not None
    assert len(rules) == 0
    assert "input" not in cleaned
    assert cleaned == row
    assert json.dumps(cleaned) == json.dumps(row)



def test_clean_row_sharegpt():
    """Test cleaning a ShareGPT row with opt-in boilerplate stripping."""
    row = {
        "conversations": [
            {"from": "human", "value": "Hello"},
            {"from": "gpt", "value": "Sure! How can I help you today?"},
        ]
    }
    cleaned, rules = clean_row(row, "sharegpt", strip_ai_boilerplate=True)
    assert cleaned is not None
    assert cleaned["conversations"][1]["value"] == "How can I help you today?"


def test_clean_row_tool_calling():
    """Test repairing JSON in tool_calls arguments with opt-in repair_json."""
    row = {
        "messages": [{"role": "user", "content": "Fetch order 123"}],
        "tool_calls": [
            {"name": "get_order", "arguments": '{"order_id": 123,}'}
        ],
    }
    cleaned, rules = clean_row(row, "tool-calling", repair_json=True)
    assert cleaned is not None
    assert "Malformed JSON in Tools" in rules
    assert cleaned["tool_calls"][0]["arguments"] == '{"order_id": 123}'


def test_clean_row_empty_dropped():
    """Test that rows with empty assistant turns are dropped by default."""
    row = {
        "messages": [
            {"role": "user", "content": "Hello"},
            {"role": "assistant", "content": "   \n\u200b   "},
        ]
    }
    cleaned, rules = clean_row(row, "chatml", min_tokens=1)
    assert cleaned is None
    assert "Empty / Whitespace Turns" in rules


def test_echo_preserved_by_default_and_pruned_with_opt_in():
    """Test that echo rows are preserved by default, and pruned only with prune_echo=True."""
    row = {
        "messages": [
            {"role": "user", "content": "Repeat after me"},
            {"role": "assistant", "content": "Repeat after me"},
        ]
    }
    # Default behavior: preserved as clean
    cleaned_default, rules_default = clean_row(row, "chatml", prune_echo=False)
    assert cleaned_default is not None
    assert len(rules_default) == 0
    assert cleaned_default["messages"][1]["content"] == "Repeat after me"

    # Opt-in behavior: dropped
    cleaned_opt_in, rules_opt_in = clean_row(row, "chatml", prune_echo=True)
    assert cleaned_opt_in is None
    assert "Target Leakage / Echo" in rules_opt_in


def test_byte_identity_clean_rows_and_arithmetic_closure():
    """Verify byte-identity for clean rows and arithmetic closure of report accounting (S1)."""
    clean_row_data = {
        "messages": [
            {"role": "user", "content": "What is 2+2?"},
            {"role": "assistant", "content": "2+2 is 4."},
        ]
    }
    single_zwsp_row = {
        "messages": [
            {"role": "user", "content": "Hello\u200b world"},
            {"role": "assistant", "content": "Hello there!"},
        ]
    }
    empty_row = {
        "messages": [
            {"role": "user", "content": "Silent question"},
            {"role": "assistant", "content": ""},
        ]
    }
    echo_row = {
        "messages": [
            {"role": "user", "content": "Echo test"},
            {"role": "assistant", "content": "Echo test"},
        ]
    }

    corpus = [clean_row_data, single_zwsp_row, empty_row, echo_row]

    # Run with default flags (all heuristics off, prune_echo off)
    cleaned_data, report = clean_dataset(corpus, "chatml")

    assert isinstance(report, CleanReport)
    assert report.total_scanned == 4
    assert report.total_clean == 2  # clean_row_data and echo_row
    assert report.total_modified == 1  # single_zwsp_row
    assert report.total_dropped == 1  # empty_row

    # Exact arithmetic closure check
    assert report.total_scanned == report.total_clean + report.total_modified + report.total_dropped

    # Rule counts sum matches or exceeds modified + dropped rules
    # (multiple rules can trigger on one row)
    assert sum(report.rule_counts.values()) >= report.total_modified + report.total_dropped
    assert all(count > 0 for count in report.rule_counts.values())

    # Output dataset preserves non-dropped rows (clean + modified) in original sequence
    assert len(cleaned_data) == 3

    # Clean rows must be 100% byte-for-byte identical to input
    assert json.dumps(cleaned_data[0]) == json.dumps(clean_row_data)
    assert json.dumps(cleaned_data[2]) == json.dumps(echo_row)

    # Modified row must have \u200b stripped (not byte-identical to dirty input)
    assert json.dumps(cleaned_data[1]) != json.dumps(single_zwsp_row)
    assert cleaned_data[1]["messages"][0]["content"] == "Hello world"

    # Modified row must not be clean and must record the exact rule
    assert "Invisible & Control Chars" in report.rule_counts
    assert report.rule_counts["Invisible & Control Chars"] == 1


def test_silently_altered_row_is_reported_modified_not_clean(monkeypatch: pytest.MonkeyPatch):
    """A row changed without any rule firing must never be reported as clean (S1)."""
    row = {
        "messages": [
            {"role": "user", "content": "What is 2+2?"},
            {"role": "assistant", "content": "2+2 is 4."},
        ],
        "train_meta": "keep",
    }

    def _drop_a_key_and_report_nothing(row_in, fmt, **kwargs):
        return {k: v for k, v in row_in.items() if k != "train_meta"}, []

    monkeypatch.setattr("soup_cli.utils.data_clean.clean_row", _drop_a_key_and_report_nothing)

    cleaned_data, report = clean_dataset([row], "chatml")

    assert "train_meta" not in cleaned_data[0], "fixture must actually alter the row"
    assert report.total_clean == 0, "an altered row was reported as clean (silent data loss)"
    assert report.total_modified == 1, "an altered row must be counted as modified"
    assert report.rule_counts == {"Format Normalization": 1}, (
        "a change no rule explains must still be attributed to Format Normalization"
    )


def test_input_file_never_modified_under_any_flags(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
    """Verify that soup data clean NEVER modifies its input file in place under any flags."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    initial_hash = _hash_file(dirty_file)

    # 1. Default live run
    out1 = tmp_path / "out1.jsonl"
    res1 = runner.invoke(app, ["data", "clean", str(dirty_file), "-o", str(out1)])
    assert res1.exit_code == 0
    assert _hash_file(dirty_file) == initial_hash

    # 2. Live run with all heuristic flags active
    out2 = tmp_path / "out2.jsonl"
    res2 = runner.invoke(
        app,
        [
            "data",
            "clean",
            str(dirty_file),
            "-o",
            str(out2),
            "--strip-boilerplate",
            "--repair-code",
            "--repair-json",
            "--prune-echo",
        ],
    )
    assert res2.exit_code == 0
    assert _hash_file(dirty_file) == initial_hash

    # 3. Dry-run mode
    res3 = runner.invoke(app, ["data", "clean", str(dirty_file), "--dry-run"])
    assert res3.exit_code == 0
    assert _hash_file(dirty_file) == initial_hash


def test_dry_run_provably_write_free(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """Verify that --dry-run creates zero files and makes zero modifications to disk."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)

    # Snapshot directory state
    dir_snapshot_before = {
        p.name: (p.stat().st_size, p.stat().st_mtime_ns) for p in tmp_path.iterdir()
    }

    result = runner.invoke(app, ["data", "clean", str(dirty_file), "--dry-run"])
    assert result.exit_code == 0

    dir_snapshot_after = {
        p.name: (p.stat().st_size, p.stat().st_mtime_ns) for p in tmp_path.iterdir()
    }

    assert dir_snapshot_before == dir_snapshot_after


def test_cli_rejects_inplace_output_overwrite(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """Verify that setting output path equal to input path is rejected."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    initial_hash = _hash_file(dirty_file)

    result = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "-o", str(dirty_file)],
    )
    assert result.exit_code == 1
    assert "Output path cannot be the same as input path" in _strip_ansi(result.output)
    assert _hash_file(dirty_file) == initial_hash


def test_cli_rejects_negative_min_tokens(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """Verify CLI rejects negative --min-tokens values."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    result = runner.invoke(app, ["data", "clean", str(dirty_file), "--min-tokens", "-5"])
    assert result.exit_code == 1
    assert "Invalid --min-tokens value" in _strip_ansi(result.output)



def test_cli_rejects_existing_output_without_force(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
    """Verify that writing to an existing output file is rejected without --force."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    existing_file = tmp_path / "existing.jsonl"
    existing_file.write_text("PRECIOUS_DATA\n", encoding="utf-8")

    # Rejected without --force
    result = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "-o", str(existing_file)],
    )
    assert result.exit_code == 1
    assert "Output file already exists" in _strip_ansi(result.output)
    assert existing_file.read_text(encoding="utf-8") == "PRECIOUS_DATA\n"

    # Accepted with --force
    result_force = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "-o", str(existing_file), "--force"],
    )
    assert result_force.exit_code == 0
    assert "PRECIOUS_DATA" not in existing_file.read_text(encoding="utf-8")


def test_cli_rejects_output_outside_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """Verify that specifying an output path outside current working directory is rejected."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    outside_output = tmp_path.parent / "outside_output.jsonl"

    result = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "-o", str(outside_output)],
    )
    assert result.exit_code == 1
    assert "Output path must be under the current working directory" in _strip_ansi(result.output)
    assert not outside_output.exists()


def test_cli_uses_atomic_write_text(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """Verify that atomic_write_text is called to perform safe atomic output writing."""
    import soup_cli.commands.data_clean as data_clean_mod

    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    output_file = tmp_path / "out_atomic.jsonl"

    calls = []
    original_atomic = data_clean_mod.atomic_write_text

    def mock_atomic(text: str, output_path: str, **kwargs):
        calls.append((text, output_path))
        return original_atomic(text, output_path, **kwargs)

    monkeypatch.setattr(data_clean_mod, "atomic_write_text", mock_atomic)

    result = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "-o", str(output_file)],
    )
    assert result.exit_code == 0
    assert len(calls) == 1
    assert calls[0][1] == str(output_file)
    assert output_file.exists()


def test_cli_preserves_echo_by_default_and_prunes_with_flag(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
    """Verify that CLI preserves echo turns by default (prune_echo=False) and prunes on flag."""
    monkeypatch.chdir(tmp_path)
    echo_file = tmp_path / "echo_dataset.jsonl"
    data = [
        {
            "messages": [
                {"role": "user", "content": "What is Python?"},
                {"role": "assistant", "content": "What is Python?"},
            ]
        }
    ]
    with open(echo_file, "w", encoding="utf-8") as f:
        f.write(json.dumps(data[0]) + "\n")

    # 1. Default invocation: preserved
    out_default = tmp_path / "echo_cleaned.jsonl"
    res_default = runner.invoke(app, ["data", "clean", str(echo_file), "-o", str(out_default)])
    assert res_default.exit_code == 0
    with open(out_default, encoding="utf-8") as f:
        lines = [json.loads(line) for line in f if line.strip()]
    assert len(lines) == 1

    # 2. Opt-in invocation with --prune-echo: pruned
    out_pruned = tmp_path / "echo_pruned.jsonl"
    res_pruned = runner.invoke(
        app,
        ["data", "clean", str(echo_file), "-o", str(out_pruned), "--prune-echo"],
    )
    assert res_pruned.exit_code == 0
    with open(out_pruned, encoding="utf-8") as f:
        lines_pruned = [json.loads(line) for line in f if line.strip()]
    assert len(lines_pruned) == 0


def test_cli_panel_clean_count_math(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """B4: Verify header panel displays report.total_clean rather than len(cleaned_data)."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    output_file = tmp_path / "out_panel.jsonl"

    result = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "-o", str(output_file)],
    )
    assert result.exit_code == 0
    output_text = _strip_ansi(result.output)

    # Scanned: 3 | Modified: 1 | Dropped: 1 | Clean: 1
    assert "Scanned: 3" in output_text
    assert "Modified: 1" in output_text
    assert "Dropped: 1" in output_text
    assert "Clean: 1" in output_text


def test_cli_clean_command_live(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """Test full soup data clean CLI command with default options."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)
    output_file = tmp_path / "cleaned_output.jsonl"

    result = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "-o", str(output_file)],
    )
    assert result.exit_code == 0
    assert output_file.exists()

    with open(output_file, encoding="utf-8") as file_handle:
        lines = [json.loads(line) for line in file_handle if line.strip()]

    # Out of 3 rows in sample:
    # 1: sanitized \u200b (modified)
    # 2: clean (clean)
    # 3: empty assistant (dropped)
    # -> 2 output rows
    assert len(lines) == 2


def test_cli_clean_command_json_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """Test soup data clean with --json flag."""
    monkeypatch.chdir(tmp_path)
    dirty_file = _create_sample_dirty_file(tmp_path)

    result = runner.invoke(
        app,
        ["data", "clean", str(dirty_file), "--dry-run", "--json"],
    )
    assert result.exit_code == 0
    payload = json.loads(_strip_ansi(result.output))
    assert payload["total_scanned"] == 3
    assert payload["output_rows"] == 2
    assert payload["total_dropped"] == 1
    assert payload["total_clean"] == 1
    assert payload["total_modified"] == 1
    assert payload["dry_run"] is True