Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Pure dataset cleaning and sanity repair engine.
Provides format-aware dataset sanitization:
1. Control character & invisible space stripping (C0 controls, zero-width spaces, CRLF)
2. Markdown code block balancing (auto-closing unclosed triple backticks)
3. Boilerplate AI disclaimer stripping ("As an AI...", "Certainly!...", "I hope this helps!")
4. JSON / tool-calling argument repair (trailing commas, unescaped quotes, markdown fences)
5. Empty and degenerate turn pruning
6. Target leakage and echo turn detection
"""
from __future__ import annotations
import json
import re
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
# C0 controls (strip under 0x20 except \t, \n, \r) + DEL (0x7F) + Unicode zero-width/BOM chars
_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\u200B\u200C\u200D\uFEFF]")
# Common LLM preamble/boilerplate prefixes in assistant turns
_BOILERPLATE_PREFIXES = [
re.compile(
r"^(?:sure(?: thing)?|certainly|of course|happy to help|absolutely)[!,.]?\s*"
r"(?:here(?:'s| is)[^:\n]*:)?\s*",
re.IGNORECASE,
),
re.compile(
r"^as an? (?:ai|artificial intelligence|large language model|assistant)[^,.\n]*[,.]?\s*"
r"(?:i (?:can|will|would be happy to|am ready to) [^,.\n]*[,.]?\s*)?",
re.IGNORECASE,
),
re.compile(
r"^i am an? (?:ai|virtual assistant|automated system)[^,.\n]*[,.]?\s*",
re.IGNORECASE,
),
]
# Common LLM canned sign-offs / suffixes in assistant turns.
#
# Run through ``.search`` on text a dataset author controls, so no pattern may
# backtrack quadratically:
# * no leading whitespace quantifier -- the caller has already ``rstrip()``ped the
# text, and with ``.search`` a leading ``\s*`` is retried from every offset inside
# one interior run of spaces (20,000 chars took ~20 s, 40,000 took ~64 s);
# * the free-text tail of a sign-off is capped at ``_SIGNOFF_TAIL_CHARS`` rather than
# left unbounded, because an unbounded tail is re-scanned to the end of the line
# from every repeat of the sign-off.
# A sign-off whose tail is longer than the cap is left alone.
_SIGNOFF_TAIL_CHARS = 200
_BOILERPLATE_SUFFIXES = [
re.compile(
r"(?:i )?hope (?:this|that) helps[!,.]?(?:\s*(?:please )?let me know if you "
r"(?:need|have) (?:anything|any(?: other)? questions)"
rf"[^.\n]{{0,{_SIGNOFF_TAIL_CHARS}}}[!,.]?)?$",
re.IGNORECASE,
),
re.compile(
r"(?:please )?let me know if you (?:have|need) (?:any )?"
r"(?:further|other|more)? (?:questions|assistance|help)[!,.]?$",
re.IGNORECASE,
),
]
# Markdown code block extractor for tool-call arguments
_MARKDOWN_JSON_RE = re.compile(r"^```(?:json)?\s*\n?([\s\S]*?)\n?```$", re.IGNORECASE)
# Regex to fix trailing commas in JSON object / array before closing brace/bracket
_TRAILING_COMMA_RE = re.compile(r",\s*([\]}])")
@dataclass
class CleanReport:
"""Statistics report summarizing actions taken across a dataset."""
total_scanned: int = 0
total_clean: int = 0
total_modified: int = 0
total_dropped: int = 0
rule_counts: Dict[str, int] = field(default_factory=dict)
def record_rule(self, rule_name: str, count: int = 1) -> None:
"""Increment count for a specific rule."""
self.rule_counts[rule_name] = self.rule_counts.get(rule_name, 0) + count
def sanitize_text(text: str) -> Tuple[str, bool]:
"""Strip control characters, zero-width spaces, and normalize CRLF/CR to LF."""
if not isinstance(text, str):
return str(text), False
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
cleaned = _CONTROL_CHARS_RE.sub("", normalized)
return cleaned, cleaned != text
def repair_code_fences(text: str) -> Tuple[str, bool]:
"""If backticks count (```) is odd, append a closing code fence."""
if not isinstance(text, str):
return text, False
fences = re.findall(r"```", text)
if len(fences) % 2 != 0:
cleaned = text.rstrip() + "\n```"
return cleaned, True
return text, False
def strip_boilerplate(text: str) -> Tuple[str, bool]:
"""Strip canned AI preamble and sign-offs from an assistant response."""
if not isinstance(text, str):
return text, False
current = text.strip()
original = current
# Strip prefix boilerplate (loop to catch chained preambles like "Certainly! As an AI...")
changed = True
passes = 0
while changed and passes < 5:
changed = False
passes += 1
for pattern in _BOILERPLATE_PREFIXES:
match = pattern.match(current)
if match:
current = current[match.end():].lstrip()
changed = True
break
# Strip suffix boilerplate
for pattern in _BOILERPLATE_SUFFIXES:
match = pattern.search(current)
if match:
current = current[: match.start()].rstrip()
break
return current, current != original
def repair_json_string(text: str) -> Tuple[str, bool]:
"""Repair common JSON corruptions in tool-call arguments.
1. Extracts JSON if wrapped in markdown ```json ... ``` fences
2. Strips trailing commas (e.g. `{"a": 1,}`)
3. Validates parseability with json.loads()
"""
if not isinstance(text, str):
return text, False
current = text.strip()
modified = False
# Check if wrapped in markdown ```json
markdown_match = _MARKDOWN_JSON_RE.match(current)
if markdown_match:
current = markdown_match.group(1).strip()
modified = True
# Check if already valid JSON
try:
json.loads(current)
return current, modified
except (json.JSONDecodeError, ValueError):
pass
# Try removing trailing commas
candidate = _TRAILING_COMMA_RE.sub(r"\1", current)
try:
json.loads(candidate)
return candidate, True
except (json.JSONDecodeError, ValueError):
pass
return text, False
def is_echo_turn(prompt: str, response: str) -> bool:
"""Check if the assistant response is merely echoing the user prompt verbatim."""
clean_p = prompt.strip().lower()
clean_r = response.strip().lower()
if not clean_p or not clean_r:
return False
return clean_p == clean_r
def _clean_arguments(
args: str, applied_rules: List[str], repair_json: bool, drop_invalid_json: bool
) -> Optional[str]:
"""Sanitize (and optionally repair) one call's JSON-string arguments.
Returns None when ``drop_invalid_json`` is set and the result does not parse.
"""
san_args, was_san = sanitize_text(args)
if was_san:
applied_rules.append("Invisible & Control Chars")
if repair_json:
repaired_args, was_repaired = repair_json_string(san_args)
if was_repaired:
applied_rules.append("Malformed JSON in Tools")
return repaired_args
if drop_invalid_json:
try:
json.loads(san_args)
except (json.JSONDecodeError, ValueError):
return None
return san_args
def _clean_tool_calls(
tool_calls: Any, applied_rules: List[str], repair_json: bool, drop_invalid_json: bool
) -> Any:
"""Clean the ``arguments`` of every call in a ``tool_calls`` list.
A call carries its arguments either in the documented ``function`` shape
(``{"function": {"name": ..., "arguments": ...}}``, what the tool-calling
loader reads) or flat (``{"name": ..., "arguments": ...}``). Arguments given
as a dict are already parsed and left alone, and a ``function`` call with no
``arguments`` is left alone too, since the loader reads it as ``{}``.
Returns None when ``drop_invalid_json`` is set and a call does not parse.
Anything that is not a list, or a call that is not a dict, is passed through.
Since None means "drop the row", callers skip a ``tool_calls`` of None (the
loader reads it as no calls) instead of passing it here.
"""
if not isinstance(tool_calls, list):
return tool_calls
cleaned_calls: List[Any] = []
for call in tool_calls:
if not isinstance(call, dict):
cleaned_calls.append(call)
continue
c = dict(call)
func = c.get("function")
if isinstance(func, dict):
args = func.get("arguments")
if isinstance(args, str):
cleaned_args = _clean_arguments(
args, applied_rules, repair_json, drop_invalid_json
)
if cleaned_args is None:
return None
c["function"] = {**func, "arguments": cleaned_args}
else:
args = c.get("arguments", "")
if isinstance(args, str):
cleaned_args = _clean_arguments(
args, applied_rules, repair_json, drop_invalid_json
)
if cleaned_args is None:
return None
c["arguments"] = cleaned_args
cleaned_calls.append(c)
return cleaned_calls
def _has_tool_calls(msg: Dict[str, Any]) -> bool:
"""True when a turn carries at least one tool call (a non-empty list)."""
calls = msg.get("tool_calls")
return isinstance(calls, list) and len(calls) > 0
def _clean_turn_tool_calls(
msg: Dict[str, Any], applied_rules: List[str], repair_json: bool, drop_invalid_json: bool
) -> bool:
"""Clean the calls an assistant turn carries, in place on ``msg`` (a copy).
Returns False when ``drop_invalid_json`` is set and a call does not parse.
"""
if msg.get("role") != "assistant" or msg.get("tool_calls") is None:
return True
cleaned_calls = _clean_tool_calls(
msg["tool_calls"], applied_rules, repair_json, drop_invalid_json
)
if cleaned_calls is None:
return False
msg["tool_calls"] = cleaned_calls
return True
def clean_row(
row: Dict[str, Any],
fmt: str,
*,
min_tokens: int = 1,
prune_echo: bool = False,
strip_ai_boilerplate: bool = False,
repair_code: bool = False,
repair_json: bool = False,
drop_invalid_json: bool = False,
) -> Tuple[Optional[Dict[str, Any]], List[str]]:
"""Clean a single row according to its format, preserving all unknown keys & roles.
Returns:
A tuple of (cleaned_row_or_None, list_of_rules_applied).
If the row should be dropped, returns (None, [drop_reason]).
"""
applied_rules: List[str] = []
if fmt == "tool-calling" or "tool_calls" in row:
cleaned_row = dict(row)
if row.get("tool_calls") is not None:
cleaned_calls = _clean_tool_calls(
row["tool_calls"], applied_rules, repair_json, drop_invalid_json
)
if cleaned_calls is None:
return None, ["Invalid JSON in Tool Calls"]
cleaned_row["tool_calls"] = cleaned_calls
if "messages" in row and isinstance(row["messages"], list):
cleaned_messages = []
for msg in row["messages"]:
if not isinstance(msg, dict):
cleaned_messages.append(msg)
continue
m = dict(msg)
content = m.get("content")
if isinstance(content, str):
san_c, was_san = sanitize_text(content)
if was_san:
applied_rules.append("Invisible & Control Chars")
m["content"] = san_c
if not _clean_turn_tool_calls(m, applied_rules, repair_json, drop_invalid_json):
return None, ["Invalid JSON in Tool Calls"]
cleaned_messages.append(m)
cleaned_row["messages"] = cleaned_messages
return cleaned_row, list(dict.fromkeys(applied_rules))
elif fmt == "chatml" or ("messages" in row and isinstance(row.get("messages"), list)):
messages = row.get("messages", [])
if not messages or not isinstance(messages, list):
return None, ["Empty / malformed messages"]
cleaned_messages = []
user_prompt = ""
for message in messages:
if not isinstance(message, dict):
cleaned_messages.append(message)
continue
msg = dict(message)
role = str(msg.get("role", "")).strip()
content = msg.get("content")
if isinstance(content, str):
sanitized_content, was_sanitized = sanitize_text(content)
if was_sanitized:
applied_rules.append("Invisible & Control Chars")
if role == "user":
user_prompt = sanitized_content
msg["content"] = sanitized_content
elif role == "assistant":
if prune_echo and user_prompt and is_echo_turn(user_prompt, sanitized_content):
return None, ["Target Leakage / Echo"]
if strip_ai_boilerplate:
sanitized_content, was_bp = strip_boilerplate(sanitized_content)
if was_bp:
applied_rules.append("Boilerplate Disclaimers")
if repair_code:
sanitized_content, was_cr = repair_code_fences(sanitized_content)
if was_cr:
applied_rules.append("Markdown Code Fence Repair")
# #1477 - a call-only turn carries its payload in tool_calls,
# and "" is what the unified tool-calling format writes there;
# keep it as the same turn with content null is kept.
if (
not _has_tool_calls(msg)
and len(sanitized_content.strip()) < min_tokens
):
return None, ["Empty / Whitespace Turns"]
msg["content"] = sanitized_content
else:
msg["content"] = sanitized_content
if not _clean_turn_tool_calls(msg, applied_rules, repair_json, drop_invalid_json):
return None, ["Invalid JSON in Tool Calls"]
cleaned_messages.append(msg)
if not any(isinstance(m, dict) and m.get("role") == "assistant" for m in cleaned_messages):
return None, ["Empty / Whitespace Turns"]
cleaned_row = dict(row)
cleaned_row["messages"] = cleaned_messages
return cleaned_row, list(dict.fromkeys(applied_rules))
elif fmt == "alpaca" or ("instruction" in row and "output" in row):
cleaned_row = dict(row)
instruction = row.get("instruction")
input_text = row.get("input")
output_text = row.get("output")
inst_san, in_san, out_san = False, False, False
san_inst, san_in, san_out = instruction, input_text, output_text
if isinstance(instruction, str):
san_inst, inst_san = sanitize_text(instruction)
if isinstance(input_text, str):
san_in, in_san = sanitize_text(input_text)
if isinstance(output_text, str):
san_out, out_san = sanitize_text(output_text)
if inst_san or in_san or out_san:
applied_rules.append("Invisible & Control Chars")
prompt_combined = (str(san_inst or "") + "\n" + str(san_in or "")).strip()
if prune_echo and is_echo_turn(prompt_combined, str(san_out or "")):
return None, ["Target Leakage / Echo"]
if strip_ai_boilerplate and isinstance(san_out, str):
san_out, was_boilerplate = strip_boilerplate(san_out)
if was_boilerplate:
applied_rules.append("Boilerplate Disclaimers")
if repair_code and isinstance(san_out, str):
san_out, was_code_repaired = repair_code_fences(san_out)
if was_code_repaired:
applied_rules.append("Markdown Code Fence Repair")
if isinstance(san_out, str) and len(san_out.strip()) < min_tokens:
return None, ["Empty / Whitespace Turns"]
if "instruction" in row:
cleaned_row["instruction"] = san_inst
if "input" in row:
cleaned_row["input"] = san_in
if "output" in row:
cleaned_row["output"] = san_out
return cleaned_row, list(dict.fromkeys(applied_rules))
elif fmt == "sharegpt" or (
"conversations" in row and isinstance(row.get("conversations"), list)
):
conversations = row.get("conversations", [])
if not conversations or not isinstance(conversations, list):
return None, ["Empty / malformed conversations"]
cleaned_convs: List[Dict[str, Any]] = []
last_human = ""
for turn in conversations:
if not isinstance(turn, dict):
cleaned_convs.append(turn)
continue
t = dict(turn)
from_role = str(t.get("from", "")).strip()
value = t.get("value")
if isinstance(value, str):
san_val, was_sanitized = sanitize_text(value)
if was_sanitized:
applied_rules.append("Invisible & Control Chars")
if from_role in ("human", "user"):
last_human = san_val
t["value"] = san_val
elif from_role in ("gpt", "assistant", "chatgpt"):
if prune_echo and last_human and is_echo_turn(last_human, san_val):
return None, ["Target Leakage / Echo"]
if strip_ai_boilerplate:
san_val, was_boilerplate = strip_boilerplate(san_val)
if was_boilerplate:
applied_rules.append("Boilerplate Disclaimers")
if repair_code:
san_val, was_code_repaired = repair_code_fences(san_val)
if was_code_repaired:
applied_rules.append("Markdown Code Fence Repair")
if len(san_val.strip()) < min_tokens:
return None, ["Empty / Whitespace Turns"]
t["value"] = san_val
else:
t["value"] = san_val
cleaned_convs.append(t)
if not any(
isinstance(t, dict) and t.get("from") in ("gpt", "assistant", "chatgpt")
for t in cleaned_convs
):
return None, ["Empty / Whitespace Turns"]
cleaned_row = dict(row)
cleaned_row["conversations"] = cleaned_convs
return cleaned_row, list(dict.fromkeys(applied_rules))
elif fmt in ("dpo", "kto"):
cleaned_row = dict(row)
prompt = row.get("prompt")
chosen = row.get("chosen")
completion = row.get("completion")
rejected = row.get("rejected")
if isinstance(prompt, str):
san_p, p_san = sanitize_text(prompt)
if p_san:
applied_rules.append("Invisible & Control Chars")
cleaned_row["prompt"] = san_p
target_chosen = chosen if "chosen" in row else completion
if isinstance(target_chosen, str):
san_c, c_san = sanitize_text(target_chosen)
if c_san:
applied_rules.append("Invisible & Control Chars")
if repair_code:
san_c, c_rep = repair_code_fences(san_c)
if c_rep:
applied_rules.append("Markdown Code Fence Repair")
if len(san_c.strip()) < min_tokens:
return None, ["Empty / Whitespace Turns"]
if "chosen" in row:
cleaned_row["chosen"] = san_c
if "completion" in row:
cleaned_row["completion"] = san_c
if isinstance(rejected, str):
san_r, r_san = sanitize_text(rejected)
if r_san:
applied_rules.append("Invisible & Control Chars")
if repair_code:
san_r, r_rep = repair_code_fences(san_r)
if r_rep:
applied_rules.append("Markdown Code Fence Repair")
cleaned_row["rejected"] = san_r
return cleaned_row, list(dict.fromkeys(applied_rules))
# Plaintext or unknown format fallback: clean control characters
cleaned_row = dict(row)
for key, val in row.items():
if isinstance(val, str):
san_val, was_san = sanitize_text(val)
if was_san:
applied_rules.append("Invisible & Control Chars")
cleaned_row[key] = san_val
return cleaned_row, list(dict.fromkeys(applied_rules))
def clean_dataset(
data: List[Dict[str, Any]],
fmt: str,
*,
min_tokens: int = 1,
prune_echo: bool = False,
strip_ai_boilerplate: bool = False,
repair_code: bool = False,
repair_json: bool = False,
drop_invalid_json: bool = False,
) -> Tuple[List[Dict[str, Any]], CleanReport]:
"""Clean a full dataset in memory and return (cleaned_rows, report)."""
report = CleanReport(total_scanned=len(data))
cleaned_rows: List[Dict[str, Any]] = []
for row in data:
cleaned, rules = clean_row(
row,
fmt,
min_tokens=min_tokens,
prune_echo=prune_echo,
strip_ai_boilerplate=strip_ai_boilerplate,
repair_code=repair_code,
repair_json=repair_json,
drop_invalid_json=drop_invalid_json,
)
if cleaned is None:
report.total_dropped += 1
for rule in rules:
report.record_rule(rule)
else:
if rules or cleaned != row:
report.total_modified += 1
if not rules:
report.record_rule("Format Normalization")
for rule in rules:
report.record_rule(rule)
else:
report.total_clean += 1
cleaned_rows.append(cleaned)
return cleaned_rows, report