Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""soup data clean — dataset cleaning and sanity repair CLI.
Applies deterministic data cleaning rules:
- Invisible control character and zero-width space sanitization
- Markdown code fence repair (closes unclosed ```)
- Canned AI preamble and disclaimer stripping ("As an AI...", "Certainly!...")
- JSON / tool-call argument repair (trailing commas, markdown wrappers)
- Empty and degenerate turn pruning
- Echo turn pruning
"""
from __future__ import annotations
import contextlib
import json
import sys
from pathlib import Path
from typing import Iterator, Optional
import typer
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from soup_cli.data.formats import detect_format
from soup_cli.data.loader import load_raw_data
from soup_cli.utils.data_clean import CleanReport, clean_dataset
from soup_cli.utils.paths import atomic_write_text, is_under_cwd
from soup_cli.utils.terminal import for_terminal
console = Console()
@contextlib.contextmanager
def _loader_warnings_to_stderr(enabled: bool) -> Iterator[None]:
"""Send the loader's warnings (a malformed JSONL line, an empty file) to stderr.
``--json`` promises a stdout that parses; the loader prints its warnings on its
own module-level console, which writes to stdout. The console is swapped for the
duration of the load and always put back.
"""
if not enabled:
yield
return
from soup_cli.data import loader
saved = loader.console
loader.console = Console(stderr=True)
try:
yield
finally:
loader.console = saved
def _render_summary_table(report: CleanReport) -> None:
"""Render the breakdown of cleaning rules applied."""
table = Table(title="Dataset Cleaning Breakdown", show_lines=True)
table.add_column("Rule / Issue", style="bold")
table.add_column("Affected Rows", justify="right")
if not report.rule_counts:
table.add_row("[green]No issues found[/]", "0")
else:
for rule_name, count in sorted(report.rule_counts.items(), key=lambda x: -x[1]):
table.add_row(for_terminal(rule_name), f"[cyan]{count}[/]")
console.print(table)
def clean(
path: str = typer.Argument(
...,
help="Path to input dataset file (JSONL)",
),
output: Optional[str] = typer.Option(
None,
"--output",
"-o",
help="Path to write cleaned dataset (default: <input>_cleaned.jsonl)",
),
force: bool = typer.Option(
False,
"--force",
"-f",
help="Overwrite output file if it already exists",
),
dry_run: bool = typer.Option(
False,
"--dry-run",
help="Preview modifications without writing output file",
),
min_tokens: int = typer.Option(
1,
"--min-tokens",
help="Minimum character/token length for assistant turns (shorter turns are dropped)",
),
prune_echo: bool = typer.Option(
False,
"--prune-echo/--no-prune-echo",
help="Prune assistant turns that merely echo the user prompt (opt-in)",
),
strip_boilerplate: bool = typer.Option(
False,
"--strip-boilerplate/--no-strip-boilerplate",
help="Strip canned AI disclaimers ('As an AI...', 'Certainly!...')",
),
repair_code: bool = typer.Option(
False,
"--repair-code/--no-repair-code",
help="Auto-close unclosed markdown ``` code fences in assistant completions",
),
repair_json: bool = typer.Option(
False,
"--repair-json/--no-repair-json",
help="Repair trailing commas and markdown fences in tool-call arguments",
),
drop_invalid_json: bool = typer.Option(
False,
"--drop-invalid-json",
help="Drop tool-calling rows with unrepairable JSON syntax errors",
),
json_output: bool = typer.Option(
False,
"--json",
help="Output machine-readable JSON summary",
),
) -> None:
"""Clean and repair a fine-tuning dataset: control characters, whitespace, format sanity."""
if min_tokens < 0:
console.print(f"[red]Invalid --min-tokens value:[/] {min_tokens} (must be >= 0)")
raise typer.Exit(1)
file_path = Path(path)
if not file_path.exists():
console.print(f"[red]File not found:[/] {file_path}")
raise typer.Exit(1)
if not is_under_cwd(file_path):
console.print(
f"[red]Input path must be under the current working directory:[/] {file_path}"
)
raise typer.Exit(1)
try:
with _loader_warnings_to_stderr(json_output):
data = load_raw_data(file_path)
except Exception as exc:
console.print(f"[red]Failed to load dataset:[/] {for_terminal(str(exc))}")
raise typer.Exit(1) from exc
if not data:
console.print("[red]Dataset is empty.[/]")
raise typer.Exit(1)
try:
fmt = detect_format(data)
except ValueError:
fmt = "plaintext"
if not json_output:
console.print(
f"[dim]Cleaning {len(data)} rows in [bold]{fmt}[/] format "
f"({'DRY RUN' if dry_run else 'LIVE'})...[/]"
)
cleaned_data, report = clean_dataset(
data,
fmt,
min_tokens=min_tokens,
prune_echo=prune_echo,
strip_ai_boilerplate=strip_boilerplate,
repair_code=repair_code,
repair_json=repair_json,
drop_invalid_json=drop_invalid_json,
)
output_path = None
if not dry_run:
if output is None:
output_path = file_path.with_name(f"{file_path.stem}_cleaned{file_path.suffix}")
else:
output_path = Path(output)
if not is_under_cwd(output_path):
console.print(
f"[red]Output path must be under the current working directory:[/] {output_path}"
)
raise typer.Exit(1)
if output_path.resolve() == file_path.resolve():
console.print(
f"[red]Output path cannot be the same as input path:[/] {output_path} "
"(soup data clean never modifies its input file in place)"
)
raise typer.Exit(1)
if output_path.exists() and not force:
console.print(
f"[red]Output file already exists:[/] {output_path} "
"(use --force / -f to overwrite)"
)
raise typer.Exit(1)
try:
jsonl_lines = [json.dumps(row, ensure_ascii=False) for row in cleaned_data]
content = "\n".join(jsonl_lines) + ("\n" if jsonl_lines else "")
atomic_write_text(content, str(output_path))
except Exception as exc:
console.print(f"[red]Failed to write output file:[/] {for_terminal(str(exc))}")
raise typer.Exit(1) from exc
if json_output:
summary_payload = {
"total_scanned": report.total_scanned,
"total_clean": report.total_clean,
"total_modified": report.total_modified,
"total_dropped": report.total_dropped,
"output_rows": len(cleaned_data),
"rule_counts": report.rule_counts,
"dry_run": dry_run,
}
sys.stdout.write(json.dumps(summary_payload, indent=2) + "\n")
return
# Render Rich header panel
header_text = (
f"Scanned: [bold]{report.total_scanned}[/] rows | "
f"Modified: [yellow]{report.total_modified}[/] | "
f"Dropped: [red]{report.total_dropped}[/] | "
f"Clean: [green]{report.total_clean}[/]"
)
console.print(Panel.fit(header_text, title="soup data clean"))
_render_summary_table(report)
if dry_run:
console.print("\n[yellow]Dry-run mode:[/] No files were written.")
return
if output_path is not None:
console.print(
f"\n[bold green]✓ Cleaned dataset saved to:[/] {output_path} "
f"({len(cleaned_data)} rows)"
)