Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

21.6 KB
Raw Blame
"""Pure dataset cleaning and sanity repair engine.

Provides format-aware dataset sanitization:
1. Control character & invisible space stripping (C0 controls, zero-width spaces, CRLF)
2. Markdown code block balancing (auto-closing unclosed triple backticks)
3. Boilerplate AI disclaimer stripping ("As an AI...", "Certainly!...", "I hope this helps!")
4. JSON / tool-calling argument repair (trailing commas, unescaped quotes, markdown fences)
5. Empty and degenerate turn pruning
6. Target leakage and echo turn detection
"""

from __future__ import annotations

import json
import re
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

# C0 controls (strip under 0x20 except \t, \n, \r) + DEL (0x7F) + Unicode zero-width/BOM chars
_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\u200B\u200C\u200D\uFEFF]")

# Common LLM preamble/boilerplate prefixes in assistant turns
_BOILERPLATE_PREFIXES = [
    re.compile(
        r"^(?:sure(?: thing)?|certainly|of course|happy to help|absolutely)[!,.]?\s*"
        r"(?:here(?:'s| is)[^:\n]*:)?\s*",
        re.IGNORECASE,
    ),
    re.compile(
        r"^as an? (?:ai|artificial intelligence|large language model|assistant)[^,.\n]*[,.]?\s*"
        r"(?:i (?:can|will|would be happy to|am ready to) [^,.\n]*[,.]?\s*)?",
        re.IGNORECASE,
    ),
    re.compile(
        r"^i am an? (?:ai|virtual assistant|automated system)[^,.\n]*[,.]?\s*",
        re.IGNORECASE,
    ),
]

# Common LLM canned sign-offs / suffixes in assistant turns.
#
# Run through ``.search`` on text a dataset author controls, so no pattern may
# backtrack quadratically:
# * no leading whitespace quantifier -- the caller has already ``rstrip()``ped the
#   text, and with ``.search`` a leading ``\s*`` is retried from every offset inside
#   one interior run of spaces (20,000 chars took ~20 s, 40,000 took ~64 s);
# * the free-text tail of a sign-off is capped at ``_SIGNOFF_TAIL_CHARS`` rather than
#   left unbounded, because an unbounded tail is re-scanned to the end of the line
#   from every repeat of the sign-off.
# A sign-off whose tail is longer than the cap is left alone.
_SIGNOFF_TAIL_CHARS = 200
_BOILERPLATE_SUFFIXES = [
    re.compile(
        r"(?:i )?hope (?:this|that) helps[!,.]?(?:\s*(?:please )?let me know if you "
        r"(?:need|have) (?:anything|any(?: other)? questions)"
        rf"[^.\n]{{0,{_SIGNOFF_TAIL_CHARS}}}[!,.]?)?$",
        re.IGNORECASE,
    ),
    re.compile(
        r"(?:please )?let me know if you (?:have|need) (?:any )?"
        r"(?:further|other|more)? (?:questions|assistance|help)[!,.]?$",
        re.IGNORECASE,
    ),
]

# Markdown code block extractor for tool-call arguments
_MARKDOWN_JSON_RE = re.compile(r"^```(?:json)?\s*\n?([\s\S]*?)\n?```$", re.IGNORECASE)

# Regex to fix trailing commas in JSON object / array before closing brace/bracket
_TRAILING_COMMA_RE = re.compile(r",\s*([\]}])")


@dataclass
class CleanReport:
    """Statistics report summarizing actions taken across a dataset."""

    total_scanned: int = 0
    total_clean: int = 0
    total_modified: int = 0
    total_dropped: int = 0
    rule_counts: Dict[str, int] = field(default_factory=dict)

    def record_rule(self, rule_name: str, count: int = 1) -> None:
        """Increment count for a specific rule."""
        self.rule_counts[rule_name] = self.rule_counts.get(rule_name, 0) + count


def sanitize_text(text: str) -> Tuple[str, bool]:
    """Strip control characters, zero-width spaces, and normalize CRLF/CR to LF."""
    if not isinstance(text, str):
        return str(text), False
    normalized = text.replace("\r\n", "\n").replace("\r", "\n")
    cleaned = _CONTROL_CHARS_RE.sub("", normalized)
    return cleaned, cleaned != text


def repair_code_fences(text: str) -> Tuple[str, bool]:
    """If backticks count (```) is odd, append a closing code fence."""
    if not isinstance(text, str):
        return text, False
    fences = re.findall(r"```", text)
    if len(fences) % 2 != 0:
        cleaned = text.rstrip() + "\n```"
        return cleaned, True
    return text, False


def strip_boilerplate(text: str) -> Tuple[str, bool]:
    """Strip canned AI preamble and sign-offs from an assistant response."""
    if not isinstance(text, str):
        return text, False
    current = text.strip()
    original = current

    # Strip prefix boilerplate (loop to catch chained preambles like "Certainly! As an AI...")
    changed = True
    passes = 0
    while changed and passes < 5:
        changed = False
        passes += 1
        for pattern in _BOILERPLATE_PREFIXES:
            match = pattern.match(current)
            if match:
                current = current[match.end():].lstrip()
                changed = True
                break

    # Strip suffix boilerplate
    for pattern in _BOILERPLATE_SUFFIXES:
        match = pattern.search(current)
        if match:
            current = current[: match.start()].rstrip()
            break

    return current, current != original


def repair_json_string(text: str) -> Tuple[str, bool]:
    """Repair common JSON corruptions in tool-call arguments.

    1. Extracts JSON if wrapped in markdown ```json ... ``` fences
    2. Strips trailing commas (e.g. `{"a": 1,}`)
    3. Validates parseability with json.loads()
    """
    if not isinstance(text, str):
        return text, False

    current = text.strip()
    modified = False

    # Check if wrapped in markdown ```json
    markdown_match = _MARKDOWN_JSON_RE.match(current)
    if markdown_match:
        current = markdown_match.group(1).strip()
        modified = True

    # Check if already valid JSON
    try:
        json.loads(current)
        return current, modified
    except (json.JSONDecodeError, ValueError):
        pass

    # Try removing trailing commas
    candidate = _TRAILING_COMMA_RE.sub(r"\1", current)
    try:
        json.loads(candidate)
        return candidate, True
    except (json.JSONDecodeError, ValueError):
        pass

    return text, False


def is_echo_turn(prompt: str, response: str) -> bool:
    """Check if the assistant response is merely echoing the user prompt verbatim."""
    clean_p = prompt.strip().lower()
    clean_r = response.strip().lower()
    if not clean_p or not clean_r:
        return False
    return clean_p == clean_r


def _clean_arguments(
    args: str, applied_rules: List[str], repair_json: bool, drop_invalid_json: bool
) -> Optional[str]:
    """Sanitize (and optionally repair) one call's JSON-string arguments.

    Returns None when ``drop_invalid_json`` is set and the result does not parse.
    """
    san_args, was_san = sanitize_text(args)
    if was_san:
        applied_rules.append("Invisible & Control Chars")
    if repair_json:
        repaired_args, was_repaired = repair_json_string(san_args)
        if was_repaired:
            applied_rules.append("Malformed JSON in Tools")
            return repaired_args
    if drop_invalid_json:
        try:
            json.loads(san_args)
        except (json.JSONDecodeError, ValueError):
            return None
    return san_args


def _clean_tool_calls(
    tool_calls: Any, applied_rules: List[str], repair_json: bool, drop_invalid_json: bool
) -> Any:
    """Clean the ``arguments`` of every call in a ``tool_calls`` list.

    A call carries its arguments either in the documented ``function`` shape
    (``{"function": {"name": ..., "arguments": ...}}``, what the tool-calling
    loader reads) or flat (``{"name": ..., "arguments": ...}``). Arguments given
    as a dict are already parsed and left alone, and a ``function`` call with no
    ``arguments`` is left alone too, since the loader reads it as ``{}``.

    Returns None when ``drop_invalid_json`` is set and a call does not parse.
    Anything that is not a list, or a call that is not a dict, is passed through.
    Since None means "drop the row", callers skip a ``tool_calls`` of None (the
    loader reads it as no calls) instead of passing it here.
    """
    if not isinstance(tool_calls, list):
        return tool_calls
    cleaned_calls: List[Any] = []
    for call in tool_calls:
        if not isinstance(call, dict):
            cleaned_calls.append(call)
            continue
        c = dict(call)
        func = c.get("function")
        if isinstance(func, dict):
            args = func.get("arguments")
            if isinstance(args, str):
                cleaned_args = _clean_arguments(
                    args, applied_rules, repair_json, drop_invalid_json
                )
                if cleaned_args is None:
                    return None
                c["function"] = {**func, "arguments": cleaned_args}
        else:
            args = c.get("arguments", "")
            if isinstance(args, str):
                cleaned_args = _clean_arguments(
                    args, applied_rules, repair_json, drop_invalid_json
                )
                if cleaned_args is None:
                    return None
                c["arguments"] = cleaned_args
        cleaned_calls.append(c)
    return cleaned_calls


def _has_tool_calls(msg: Dict[str, Any]) -> bool:
    """True when a turn carries at least one tool call (a non-empty list)."""
    calls = msg.get("tool_calls")
    return isinstance(calls, list) and len(calls) > 0


def _clean_turn_tool_calls(
    msg: Dict[str, Any], applied_rules: List[str], repair_json: bool, drop_invalid_json: bool
) -> bool:
    """Clean the calls an assistant turn carries, in place on ``msg`` (a copy).

    Returns False when ``drop_invalid_json`` is set and a call does not parse.
    """
    if msg.get("role") != "assistant" or msg.get("tool_calls") is None:
        return True
    cleaned_calls = _clean_tool_calls(
        msg["tool_calls"], applied_rules, repair_json, drop_invalid_json
    )
    if cleaned_calls is None:
        return False
    msg["tool_calls"] = cleaned_calls
    return True


def clean_row(
    row: Dict[str, Any],
    fmt: str,
    *,
    min_tokens: int = 1,
    prune_echo: bool = False,
    strip_ai_boilerplate: bool = False,
    repair_code: bool = False,
    repair_json: bool = False,
    drop_invalid_json: bool = False,
) -> Tuple[Optional[Dict[str, Any]], List[str]]:
    """Clean a single row according to its format, preserving all unknown keys & roles.

    Returns:
        A tuple of (cleaned_row_or_None, list_of_rules_applied).
        If the row should be dropped, returns (None, [drop_reason]).
    """
    applied_rules: List[str] = []

    if fmt == "tool-calling" or "tool_calls" in row:
        cleaned_row = dict(row)
        if row.get("tool_calls") is not None:
            cleaned_calls = _clean_tool_calls(
                row["tool_calls"], applied_rules, repair_json, drop_invalid_json
            )
            if cleaned_calls is None:
                return None, ["Invalid JSON in Tool Calls"]
            cleaned_row["tool_calls"] = cleaned_calls

        if "messages" in row and isinstance(row["messages"], list):
            cleaned_messages = []
            for msg in row["messages"]:
                if not isinstance(msg, dict):
                    cleaned_messages.append(msg)
                    continue
                m = dict(msg)
                content = m.get("content")
                if isinstance(content, str):
                    san_c, was_san = sanitize_text(content)
                    if was_san:
                        applied_rules.append("Invisible & Control Chars")
                    m["content"] = san_c
                if not _clean_turn_tool_calls(m, applied_rules, repair_json, drop_invalid_json):
                    return None, ["Invalid JSON in Tool Calls"]
                cleaned_messages.append(m)
            cleaned_row["messages"] = cleaned_messages

        return cleaned_row, list(dict.fromkeys(applied_rules))

    elif fmt == "chatml" or ("messages" in row and isinstance(row.get("messages"), list)):
        messages = row.get("messages", [])
        if not messages or not isinstance(messages, list):
            return None, ["Empty / malformed messages"]

        cleaned_messages = []
        user_prompt = ""

        for message in messages:
            if not isinstance(message, dict):
                cleaned_messages.append(message)
                continue

            msg = dict(message)
            role = str(msg.get("role", "")).strip()
            content = msg.get("content")

            if isinstance(content, str):
                sanitized_content, was_sanitized = sanitize_text(content)
                if was_sanitized:
                    applied_rules.append("Invisible & Control Chars")

                if role == "user":
                    user_prompt = sanitized_content
                    msg["content"] = sanitized_content
                elif role == "assistant":
                    if prune_echo and user_prompt and is_echo_turn(user_prompt, sanitized_content):
                        return None, ["Target Leakage / Echo"]

                    if strip_ai_boilerplate:
                        sanitized_content, was_bp = strip_boilerplate(sanitized_content)
                        if was_bp:
                            applied_rules.append("Boilerplate Disclaimers")

                    if repair_code:
                        sanitized_content, was_cr = repair_code_fences(sanitized_content)
                        if was_cr:
                            applied_rules.append("Markdown Code Fence Repair")

                    # #1477 - a call-only turn carries its payload in tool_calls,
                    # and "" is what the unified tool-calling format writes there;
                    # keep it as the same turn with content null is kept.
                    if (
                        not _has_tool_calls(msg)
                        and len(sanitized_content.strip()) < min_tokens
                    ):
                        return None, ["Empty / Whitespace Turns"]

                    msg["content"] = sanitized_content
                else:
                    msg["content"] = sanitized_content
            if not _clean_turn_tool_calls(msg, applied_rules, repair_json, drop_invalid_json):
                return None, ["Invalid JSON in Tool Calls"]
            cleaned_messages.append(msg)

        if not any(isinstance(m, dict) and m.get("role") == "assistant" for m in cleaned_messages):
            return None, ["Empty / Whitespace Turns"]

        cleaned_row = dict(row)
        cleaned_row["messages"] = cleaned_messages
        return cleaned_row, list(dict.fromkeys(applied_rules))

    elif fmt == "alpaca" or ("instruction" in row and "output" in row):
        cleaned_row = dict(row)
        instruction = row.get("instruction")
        input_text = row.get("input")
        output_text = row.get("output")

        inst_san, in_san, out_san = False, False, False
        san_inst, san_in, san_out = instruction, input_text, output_text

        if isinstance(instruction, str):
            san_inst, inst_san = sanitize_text(instruction)
        if isinstance(input_text, str):
            san_in, in_san = sanitize_text(input_text)
        if isinstance(output_text, str):
            san_out, out_san = sanitize_text(output_text)

        if inst_san or in_san or out_san:
            applied_rules.append("Invisible & Control Chars")

        prompt_combined = (str(san_inst or "") + "\n" + str(san_in or "")).strip()
        if prune_echo and is_echo_turn(prompt_combined, str(san_out or "")):
            return None, ["Target Leakage / Echo"]

        if strip_ai_boilerplate and isinstance(san_out, str):
            san_out, was_boilerplate = strip_boilerplate(san_out)
            if was_boilerplate:
                applied_rules.append("Boilerplate Disclaimers")

        if repair_code and isinstance(san_out, str):
            san_out, was_code_repaired = repair_code_fences(san_out)
            if was_code_repaired:
                applied_rules.append("Markdown Code Fence Repair")

        if isinstance(san_out, str) and len(san_out.strip()) < min_tokens:
            return None, ["Empty / Whitespace Turns"]

        if "instruction" in row:
            cleaned_row["instruction"] = san_inst
        if "input" in row:
            cleaned_row["input"] = san_in
        if "output" in row:
            cleaned_row["output"] = san_out
        return cleaned_row, list(dict.fromkeys(applied_rules))

    elif fmt == "sharegpt" or (
        "conversations" in row and isinstance(row.get("conversations"), list)
    ):
        conversations = row.get("conversations", [])
        if not conversations or not isinstance(conversations, list):
            return None, ["Empty / malformed conversations"]

        cleaned_convs: List[Dict[str, Any]] = []
        last_human = ""

        for turn in conversations:
            if not isinstance(turn, dict):
                cleaned_convs.append(turn)
                continue

            t = dict(turn)
            from_role = str(t.get("from", "")).strip()
            value = t.get("value")

            if isinstance(value, str):
                san_val, was_sanitized = sanitize_text(value)
                if was_sanitized:
                    applied_rules.append("Invisible & Control Chars")

                if from_role in ("human", "user"):
                    last_human = san_val
                    t["value"] = san_val
                elif from_role in ("gpt", "assistant", "chatgpt"):
                    if prune_echo and last_human and is_echo_turn(last_human, san_val):
                        return None, ["Target Leakage / Echo"]

                    if strip_ai_boilerplate:
                        san_val, was_boilerplate = strip_boilerplate(san_val)
                        if was_boilerplate:
                            applied_rules.append("Boilerplate Disclaimers")

                    if repair_code:
                        san_val, was_code_repaired = repair_code_fences(san_val)
                        if was_code_repaired:
                            applied_rules.append("Markdown Code Fence Repair")

                    if len(san_val.strip()) < min_tokens:
                        return None, ["Empty / Whitespace Turns"]

                    t["value"] = san_val
                else:
                    t["value"] = san_val
            cleaned_convs.append(t)

        if not any(
            isinstance(t, dict) and t.get("from") in ("gpt", "assistant", "chatgpt")
            for t in cleaned_convs
        ):
            return None, ["Empty / Whitespace Turns"]

        cleaned_row = dict(row)
        cleaned_row["conversations"] = cleaned_convs
        return cleaned_row, list(dict.fromkeys(applied_rules))

    elif fmt in ("dpo", "kto"):
        cleaned_row = dict(row)
        prompt = row.get("prompt")
        chosen = row.get("chosen")
        completion = row.get("completion")
        rejected = row.get("rejected")

        if isinstance(prompt, str):
            san_p, p_san = sanitize_text(prompt)
            if p_san:
                applied_rules.append("Invisible & Control Chars")
            cleaned_row["prompt"] = san_p

        target_chosen = chosen if "chosen" in row else completion
        if isinstance(target_chosen, str):
            san_c, c_san = sanitize_text(target_chosen)
            if c_san:
                applied_rules.append("Invisible & Control Chars")
            if repair_code:
                san_c, c_rep = repair_code_fences(san_c)
                if c_rep:
                    applied_rules.append("Markdown Code Fence Repair")
            if len(san_c.strip()) < min_tokens:
                return None, ["Empty / Whitespace Turns"]
            if "chosen" in row:
                cleaned_row["chosen"] = san_c
            if "completion" in row:
                cleaned_row["completion"] = san_c

        if isinstance(rejected, str):
            san_r, r_san = sanitize_text(rejected)
            if r_san:
                applied_rules.append("Invisible & Control Chars")
            if repair_code:
                san_r, r_rep = repair_code_fences(san_r)
                if r_rep:
                    applied_rules.append("Markdown Code Fence Repair")
            cleaned_row["rejected"] = san_r

        return cleaned_row, list(dict.fromkeys(applied_rules))

    # Plaintext or unknown format fallback: clean control characters
    cleaned_row = dict(row)
    for key, val in row.items():
        if isinstance(val, str):
            san_val, was_san = sanitize_text(val)
            if was_san:
                applied_rules.append("Invisible & Control Chars")
            cleaned_row[key] = san_val

    return cleaned_row, list(dict.fromkeys(applied_rules))


def clean_dataset(
    data: List[Dict[str, Any]],
    fmt: str,
    *,
    min_tokens: int = 1,
    prune_echo: bool = False,
    strip_ai_boilerplate: bool = False,
    repair_code: bool = False,
    repair_json: bool = False,
    drop_invalid_json: bool = False,
) -> Tuple[List[Dict[str, Any]], CleanReport]:
    """Clean a full dataset in memory and return (cleaned_rows, report)."""
    report = CleanReport(total_scanned=len(data))
    cleaned_rows: List[Dict[str, Any]] = []

    for row in data:
        cleaned, rules = clean_row(
            row,
            fmt,
            min_tokens=min_tokens,
            prune_echo=prune_echo,
            strip_ai_boilerplate=strip_ai_boilerplate,
            repair_code=repair_code,
            repair_json=repair_json,
            drop_invalid_json=drop_invalid_json,
        )
        if cleaned is None:
            report.total_dropped += 1
            for rule in rules:
                report.record_rule(rule)
        else:
            if rules or cleaned != row:
                report.total_modified += 1
                if not rules:
                    report.record_rule("Format Normalization")
                for rule in rules:
                    report.record_rule(rule)
            else:
                report.total_clean += 1
            cleaned_rows.append(cleaned)

    return cleaned_rows, report