Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for Issue #1231: soup expect PII and refusal checks across all dataset formats."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Mapping
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.data.formats import FORMAT_SIGNATURES
from soup_cli.utils.expectations import (
FORMAT_EXTRACTION_RULES,
expect_no_pii,
expect_no_refusal_pattern,
expect_token_length_between,
)
from tests.conftest import strip_ansi
PII_SAMPLE = "Contact [email protected], SSN 123-45-6789."
REFUSAL_SAMPLE = "I'm sorry, but I can't help with that."
# -----------------------------------------------------------------------------
# 1. PII extraction across all format signatures and text-bearing fields
# -----------------------------------------------------------------------------
ALL_FORMAT_PII_CASES = [
("plaintext_text", {"text": PII_SAMPLE}),
("chatml_user", {"messages": [{"role": "user", "content": PII_SAMPLE}]}),
("chatml_assistant", {"messages": [{"role": "assistant", "content": PII_SAMPLE}]}),
("chatml_system", {"messages": [{"role": "system", "content": PII_SAMPLE}]}),
(
"chatml_content_parts",
{"messages": [{"role": "user", "content": [{"type": "text", "text": PII_SAMPLE}]}]},
),
("alpaca_instruction", {"instruction": PII_SAMPLE, "output": "ok"}),
("alpaca_input", {"instruction": "Summarize", "input": PII_SAMPLE, "output": "ok"}),
("alpaca_output", {"instruction": "Summarize", "output": PII_SAMPLE}),
("alpaca_system", {"instruction": "Summarize", "system": PII_SAMPLE, "output": "ok"}),
(
"sharegpt_human",
{"conversations": [{"from": "human", "value": PII_SAMPLE}, {"from": "gpt", "value": "ok"}]},
),
(
"sharegpt_gpt",
{"conversations": [{"from": "human", "value": "x"}, {"from": "gpt", "value": PII_SAMPLE}]},
),
(
"llava",
{
"image": "img.png",
"conversations": [
{"from": "human", "value": PII_SAMPLE},
{"from": "gpt", "value": "ok"},
],
},
),
(
"sharegpt4v",
{
"image": "img.png",
"conversations": [
{"from": "human", "value": PII_SAMPLE},
{"from": "gpt", "value": "ok"},
],
},
),
("dpo_prompt", {"prompt": PII_SAMPLE, "chosen": "c", "rejected": "r"}),
("dpo_chosen_str", {"prompt": "p", "chosen": PII_SAMPLE, "rejected": "r"}),
("dpo_rejected_str", {"prompt": "p", "chosen": "c", "rejected": PII_SAMPLE}),
(
"dpo_chosen_conversational",
{
"prompt": "p",
"chosen": [{"role": "assistant", "content": PII_SAMPLE}],
"rejected": [{"role": "assistant", "content": "r"}],
},
),
(
"dpo_rejected_conversational",
{
"prompt": "p",
"chosen": [{"role": "assistant", "content": "c"}],
"rejected": [{"role": "assistant", "content": PII_SAMPLE}],
},
),
("kto_prompt", {"prompt": PII_SAMPLE, "completion": "ok", "label": True}),
("kto_completion", {"prompt": "p", "completion": PII_SAMPLE, "label": True}),
("embedding_anchor_str", {"anchor": PII_SAMPLE, "positive": "pos"}),
("embedding_positive_str", {"anchor": "anc", "positive": PII_SAMPLE}),
("embedding_negative_str", {"anchor": "anc", "positive": "pos", "negative": PII_SAMPLE}),
("embedding_anchor_list", {"anchor": [PII_SAMPLE], "positive": "pos"}),
("embedding_positive_list", {"anchor": "anc", "positive": [PII_SAMPLE]}),
("embedding_negative_list", {"anchor": "anc", "positive": "pos", "negative": [PII_SAMPLE]}),
("audio", {"audio": "a.wav", "messages": [{"role": "user", "content": PII_SAMPLE}]}),
("asr", {"audio": "a.wav", "text": PII_SAMPLE}),
(
"tool_calling_messages",
{"messages": [{"role": "user", "content": PII_SAMPLE}], "tools": [], "tool_calls": []},
),
(
"tool_calling_tools_desc",
{
"messages": [{"role": "user", "content": "x"}],
"tools": [{"type": "function", "function": {"name": "f", "description": PII_SAMPLE}}],
"tool_calls": [],
},
),
(
"tool_calling_tool_calls_str",
{
"messages": [{"role": "assistant", "content": "run"}],
"tools": [],
"tool_calls": [
{"function": {"name": "f", "arguments": f'{{"email": "{PII_SAMPLE}"}}'}}
],
},
),
(
"tool_calling_tool_calls_dict",
{
"messages": [{"role": "assistant", "content": "run"}],
"tools": [],
"tool_calls": [{"function": {"name": "f", "arguments": {"contact": PII_SAMPLE}}}],
},
),
(
"tool_calling_message_tool_calls",
{
"messages": [
{
"role": "assistant",
"content": "run",
"tool_calls": [{"function": {"name": "f", "arguments": PII_SAMPLE}}],
}
],
"tools": [],
"tool_calls": [],
},
),
]
@pytest.mark.parametrize(
"case_name,row", ALL_FORMAT_PII_CASES, ids=[c[0] for c in ALL_FORMAT_PII_CASES]
)
def test_expect_no_pii_all_formats(case_name: str, row: Mapping[str, Any]) -> None:
res = expect_no_pii([row])
assert res.passed is False, f"Expected PII failure for {case_name}, but passed: {res}"
assert res.num_violations == 1
assert any("PII detected" in d for d in res.details)
# -----------------------------------------------------------------------------
# 2. Assistant refusal detection across formats
# -----------------------------------------------------------------------------
def test_expect_no_refusal_pattern_sharegpt_gpt() -> None:
row = {
"conversations": [
{"from": "human", "value": "How do I do X?"},
{"from": "gpt", "value": REFUSAL_SAMPLE},
]
}
res = expect_no_refusal_pattern([row])
assert res.passed is False
assert res.num_violations == 1
assert "refusal pattern matched" in res.details[0]
def test_expect_no_refusal_pattern_sharegpt_assistant() -> None:
row = {
"conversations": [
{"from": "human", "value": "How do I do X?"},
{"from": "assistant", "value": REFUSAL_SAMPLE},
]
}
res = expect_no_refusal_pattern([row])
assert res.passed is False
assert res.num_violations == 1
assert "refusal pattern matched" in res.details[0]
def test_expect_no_refusal_pattern_dpo_chosen_string() -> None:
row = {
"prompt": "How do I bake bread?",
"chosen": REFUSAL_SAMPLE,
"rejected": "Mix flour and water...",
}
res = expect_no_refusal_pattern([row])
assert res.passed is False
assert res.num_violations == 1
assert "refusal pattern matched" in res.details[0]
def test_expect_no_refusal_pattern_dpo_chosen_messages() -> None:
row = {
"prompt": "How do I bake bread?",
"chosen": [{"role": "assistant", "content": REFUSAL_SAMPLE}],
"rejected": [{"role": "assistant", "content": "Mix flour and water..."}],
}
res = expect_no_refusal_pattern([row])
assert res.passed is False
assert res.num_violations == 1
assert "refusal pattern matched" in res.details[0]
def test_expect_no_refusal_pattern_kto_completion() -> None:
row = {
"prompt": "How do I bake bread?",
"completion": REFUSAL_SAMPLE,
"label": True,
}
res = expect_no_refusal_pattern([row])
assert res.passed is False
assert res.num_violations == 1
assert "refusal pattern matched" in res.details[0]
def test_expect_no_refusal_pattern_controls() -> None:
# User message contains refusal words -> should NOT fail assistant refusal check.
user_refusal = {
"conversations": [
{"from": "human", "value": REFUSAL_SAMPLE},
{"from": "gpt", "value": "Sure, here is how you do it."},
]
}
res = expect_no_refusal_pattern([user_refusal])
assert res.passed is True
assert res.num_violations == 0
# DPO rejected contains refusal -> should NOT fail (only chosen is assistant target).
rejected_refusal = {
"prompt": "How do I bake bread?",
"chosen": "Mix flour and water...",
"rejected": REFUSAL_SAMPLE,
}
res = expect_no_refusal_pattern([rejected_refusal])
assert res.passed is True
assert res.num_violations == 0
# -----------------------------------------------------------------------------
# 3. Token length counting across formats
# -----------------------------------------------------------------------------
def test_expect_token_length_between_across_formats() -> None:
text_9_words = "one two three four five six seven eight nine"
rows = [
{"text": text_9_words},
{"conversations": [{"from": "human", "value": text_9_words}]},
{"instruction": text_9_words, "output": ""},
{"messages": [{"role": "user", "content": text_9_words}]},
{"prompt": text_9_words, "chosen": "", "rejected": ""},
{"prompt": text_9_words, "completion": "", "label": True},
{"anchor": text_9_words, "positive": ""},
{"audio": "audio.wav", "text": text_9_words},
]
for i, row in enumerate(rows):
res = expect_token_length_between([row], min_tokens=9, max_tokens=9)
assert res.passed is True, f"Row {i} ({row}) failed 9-word length check: {res}"
res_fail = expect_token_length_between([row], min_tokens=10, max_tokens=20)
assert res_fail.passed is False, f"Row {i} should fail [10, 20]: {res_fail}"
assert res_fail.num_violations == 1
assert "9 tokens" in res_fail.details[0]
# -----------------------------------------------------------------------------
# 4. Fail-closed on rows with no extractable text
# -----------------------------------------------------------------------------
def test_no_extractable_text_fails_closed() -> None:
unextractable_rows = [
{},
{"image": "photo.png"},
{"count": 42},
]
pii_res = expect_no_pii(unextractable_rows)
assert pii_res.passed is False
assert pii_res.num_violations == 3
assert all("no extractable text" in d for d in pii_res.details)
length_res = expect_token_length_between(unextractable_rows, min_tokens=1, max_tokens=100)
assert length_res.passed is False
assert length_res.num_violations == 3
assert all("no extractable text" in d for d in length_res.details)
refusal_res = expect_no_refusal_pattern(unextractable_rows)
assert refusal_res.passed is False
assert refusal_res.num_violations == 3
assert all("no extractable text" in d for d in refusal_res.details)
# -----------------------------------------------------------------------------
# 5. Ratchet test: FORMAT_SIGNATURES vs FORMAT_EXTRACTION_RULES
# -----------------------------------------------------------------------------
def test_ratchet_all_format_signatures_have_extraction_rules() -> None:
for fmt_name in FORMAT_SIGNATURES:
assert fmt_name in FORMAT_EXTRACTION_RULES, (
f"Format {fmt_name!r} in FORMAT_SIGNATURES lacks rules in FORMAT_EXTRACTION_RULES"
)
rule = FORMAT_EXTRACTION_RULES[fmt_name]
assert "text_fields" in rule
assert "assistant_fields" in rule
assert isinstance(rule["text_fields"], tuple)
assert isinstance(rule["assistant_fields"], tuple)
# -----------------------------------------------------------------------------
# 6. CLI reproduction test from Issue #1231 exits 2
# -----------------------------------------------------------------------------
def test_cli_expect_flags_pii_and_refusal_and_exits_2(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.chdir(tmp_path)
# 4 rows from the issue: ShareGPT (PII), DPO (PII), Alpaca input (PII), DPO (refusal).
data_file = tmp_path / "data.jsonl"
data_rows = [
{"conversations": [{"from": "human", "value": PII_SAMPLE}, {"from": "gpt", "value": "ok"}]},
{"prompt": "hi", "chosen": PII_SAMPLE, "rejected": "no"},
{"instruction": "Summarize", "input": PII_SAMPLE, "output": "ok"},
{"prompt": "How do I bake bread?", "chosen": REFUSAL_SAMPLE, "rejected": "Mix flour..."},
]
with data_file.open("w", encoding="utf-8") as f:
for r in data_rows:
f.write(json.dumps(r) + "\n")
suite_file = tmp_path / "suite.yaml"
suite_yaml = """
expectations:
- name: expect_no_pii
- name: expect_no_refusal_pattern
"""
suite_file.write_text(suite_yaml, encoding="utf-8")
runner = CliRunner()
result = runner.invoke(app, ["expect", str(data_file), str(suite_file)])
assert result.exit_code == 2, (
f"Expected exit code 2, got {result.exit_code}. Output:\n{result.output}"
)
clean_out = " ".join(strip_ansi(result.output).split())
assert "expect_no_pii" in clean_out
assert "expect_no_refusal_pattern" in clean_out
assert "FAIL" in clean_out
assert "Violations: expect_no_pii" in clean_out
assert "Violations: expect_no_refusal_pattern" in clean_out
# -----------------------------------------------------------------------------
# 7. Comprehensive matrix: every field of every format scanned
# -----------------------------------------------------------------------------
NON_TEXT_FIELDS = frozenset({"label", "image", "audio"})
_SHAPES = {
"conversations": lambda t: [{"from": "human", "value": "q"}, {"from": "gpt", "value": t}],
"messages": lambda t: [{"role": "user", "content": "q"}, {"role": "assistant", "content": t}],
"tools": lambda t: [{"type": "function", "function": {"name": "f", "description": t}}],
"tool_calls": lambda t: [{"function": {"name": "f", "arguments": t}}],
"label": lambda t: True,
"image": lambda t: "img.png",
"audio": lambda t: "a.wav",
"completions": lambda t: ["step one", t],
"distractor_docs": lambda t: ["another doc", t],
"segments": lambda t: [{"text": "prefix", "label": False}, {"text": t, "label": True}],
}
def _row(fmt: str, field: str, payload: str) -> dict:
keys = (
set(FORMAT_SIGNATURES.get(fmt, ()))
| set(FORMAT_EXTRACTION_RULES[fmt]["text_fields"])
| {field}
)
return {
key: _SHAPES.get(key, lambda t: t)(payload if key == field else "plain words")
for key in sorted(keys)
}
def _text_fields(fmt: str) -> list:
declared = set(FORMAT_EXTRACTION_RULES[fmt]["text_fields"])
return sorted(declared | (set(FORMAT_SIGNATURES.get(fmt, ())) - NON_TEXT_FIELDS))
PII_CASES = [
(fmt, field) for fmt in sorted(FORMAT_EXTRACTION_RULES) for field in _text_fields(fmt)
]
REFUSAL_CASES = [
(fmt, field)
for fmt in sorted(FORMAT_EXTRACTION_RULES)
for field in FORMAT_EXTRACTION_RULES[fmt]["assistant_fields"]
]
def test_every_declared_rule_is_non_empty() -> None:
for fmt, rule in FORMAT_EXTRACTION_RULES.items():
assert rule["text_fields"], fmt
if fmt != "embedding":
assert rule["assistant_fields"], fmt
def test_unlabelled_input_output_segment_is_not_an_assistant_refusal() -> None:
row = {
"segments": [
{"text": REFUSAL_SAMPLE, "label": False},
{"text": "Sure, here is how.", "label": True},
]
}
assert expect_no_refusal_pattern([row]).passed is True
@pytest.mark.parametrize("fmt,field", PII_CASES, ids=[f"{f}.{k}" for f, k in PII_CASES])
def test_every_text_field_of_every_format_is_scanned(fmt: str, field: str) -> None:
res = expect_no_pii([_row(fmt, field, PII_SAMPLE)])
assert any("PII detected" in d for d in res.details), (fmt, field, res)
@pytest.mark.parametrize("fmt,field", REFUSAL_CASES, ids=[f"{f}.{k}" for f, k in REFUSAL_CASES])
def test_every_assistant_field_of_every_format_is_scanned(fmt: str, field: str) -> None:
res = expect_no_refusal_pattern([_row(fmt, field, REFUSAL_SAMPLE)])
assert any("refusal pattern matched" in d for d in res.details), (fmt, field, res)
@pytest.mark.parametrize(
"row",
[
{"prompt": "What is 2+2?"},
{"messages": [{"role": "user", "content": "What is 2+2?"}]},
{"conversations": [{"from": "human", "value": "What is 2+2?"}]},
{"anchor": "a cat", "positive": "a feline"},
],
ids=["prompt_only", "chatml_user_only", "sharegpt_human_only", "embedding"],
)
def test_rows_without_an_assistant_turn_are_not_refusal_violations(row: dict) -> None:
assert expect_no_refusal_pattern([row]).passed is True
def test_refusal_in_a_user_message_is_not_an_assistant_refusal() -> None:
row = {
"messages": [
{"role": "user", "content": REFUSAL_SAMPLE},
{"role": "assistant", "content": "Sure, here is how."},
]
}
assert expect_no_refusal_pattern([row]).passed is True
def test_pii_after_a_long_field_is_still_found() -> None:
row = {"instruction": "Summarise", "input": "lorem ipsum " * 5000, "output": PII_SAMPLE}
res = expect_no_pii([row])
assert any("PII detected" in d for d in res.details), res
def test_plaintext_assistant_refusal_flags_and_ok_passes() -> None:
assert expect_no_refusal_pattern([{"text": REFUSAL_SAMPLE}]).passed is False
assert expect_no_refusal_pattern([{"text": "ok"}]).passed is True
def test_raft_and_prm_formats_are_scanned_properly() -> None:
raft_clean = {
"query": "What is Python?",
"golden_doc": "Python is a language.",
"distractor_docs": ["Cats are pets."],
"answer": "A programming language.",
}
assert expect_no_pii([raft_clean]).passed is True
assert expect_no_refusal_pattern([raft_clean]).passed is True
raft_refusal = dict(raft_clean, answer=REFUSAL_SAMPLE)
res_refusal = expect_no_refusal_pattern([raft_refusal])
assert res_refusal.passed is False
assert "refusal pattern matched" in res_refusal.details[0]
prm_row_pii = {
"prompt": "What is 2+2?",
"completions": [f"Step 1: {PII_SAMPLE}"],
"labels": [True],
}
res_prm_pii = expect_no_pii([prm_row_pii])
assert res_prm_pii.passed is False
assert any("PII detected" in d for d in res_prm_pii.details)
def test_chosen_user_message_refusal_does_not_flag_assistant_refusal() -> None:
row = {
"prompt": "p",
"chosen": [{"role": "user", "content": REFUSAL_SAMPLE}],
"rejected": "r",
}
assert expect_no_refusal_pattern([row]).passed is True