Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
"""
Detect None/empty content turns in conversation datasets. Reports findings
without modifying data.
Usage:
from .dataset_none_detect import scan_dataset, print_report
stats = scan_dataset(dataset) # auto-detect + scan
stats = scan_dataset(dataset, fmt="chatml") # explicit format
print_report(stats, stats["format"])
Dependencies: only `datasets` (already in studio/unsloth) + stdlib.
Supported formats (via FORMAT_REGISTRY):
alpaca instruction/output instruction + output must be set
chatml messages/conversations/texts role + content per turn
sharegpt conversations from/value per turn
gptoss messages (alias: gpt-oss) role/content; has a developer turn
Any role/content chat template matches chatml, so new templates need no change;
add a FORMAT_REGISTRY entry only for a genuinely new column/turn shape.
"""
from datasets import Dataset
# Candidate conversation column names, checked in priority order; shared by detection and scanning.
CONVERSATION_COLUMNS = ("messages", "conversations", "texts")
# Minimum turn key sets identifying a column as conversational (not e.g. messages=[{"id":1}]).
_CHAT_KEY_SETS = (frozenset({"role", "content"}), frozenset({"from", "value"}))
def _probe_conversation(dataset: Dataset, candidates = None):
"""Returns the conversation column with its turn keys and roles seen, or None when no column matches."""
if candidates is None:
candidates = CONVERSATION_COLUMNS
columns = set(dataset.column_names)
# Remember the first all-corrupt candidate.
all_corrupt_fallback = None
for col in candidates:
if col not in columns:
continue
# Scan up to 100 rows - row 0 alone may be empty/malformed.
first = None
for i in range(min(len(dataset), 100)):
sample = dataset[i][col]
if not isinstance(sample, list) or len(sample) == 0:
continue
# Skip non-dict leading turns (e.g. [None, {"role": ...}]).
first_turn = next((t for t in sample if isinstance(t, dict)), None)
if first_turn is not None:
first = first_turn
break
if first is None:
# All-corrupt fallback is recorded only if turn-shaped; a later plausible candidate upgrades it.
if all_corrupt_fallback is None or not all_corrupt_fallback.get("has_plausible_turns"):
has_plausible_turns = False
for i in range(min(len(dataset), 100)):
cell = dataset[i][col]
if cell is None:
has_plausible_turns = True
break
# Single-dict (struct) cells are metadata, not chat; leave them unknown, as format_detection.py.
if isinstance(cell, list):
# Plausible only if the list holds a dict/None turn; empty lists and list-of-strings are not chat.
if any(t is None or isinstance(t, dict) for t in cell):
has_plausible_turns = True
break
all_corrupt_fallback = {
"column": col,
"turn_keys": set(),
"roles": set(),
"all_corrupt": True,
"has_plausible_turns": has_plausible_turns,
}
continue
# Use the same 100-row window to gather keys/roles.
turn_keys = set()
roles = set()
for i in range(min(len(dataset), 100)):
conv = dataset[i][col]
if isinstance(conv, list):
for t in conv:
if isinstance(t, dict):
turn_keys.update(t.keys())
r = t.get("role") or t.get("from")
if r:
roles.add(str(r))
# Lacking a chat key pair, a conversational key marks corrupt-but-real chat; pure metadata does not.
_CONV_KEYS = {"role", "from", "content", "value"}
if not any(keys <= turn_keys for keys in _CHAT_KEY_SETS):
schema_less_plausible = bool(turn_keys & _CONV_KEYS)
if all_corrupt_fallback is None or not all_corrupt_fallback.get("has_plausible_turns"):
all_corrupt_fallback = {
"column": col,
"turn_keys": turn_keys,
"roles": roles,
"all_corrupt": True,
"has_plausible_turns": schema_less_plausible,
}
continue
return {"column": col, "turn_keys": turn_keys, "roles": roles}
# No healthy column found; return the all_corrupt fallback if any.
return all_corrupt_fallback
# None-detection helpers
def is_none_or_empty(value) -> bool:
"""True if value is None, empty string, whitespace-only, or an empty/whitespace-only VLM content block list."""
if value is None:
return True
if isinstance(value, str):
# Zero-width/BOM chars count as empty; the two-pass strip (ws, invisibles, ws) catches mixed cases like a zero-width space next to a space.
stripped = value.strip().strip("\ufeff\u200b\u200c\u200d\u2060").strip()
if not stripped:
return True
if isinstance(value, list):
# A non-text block is real content: an image-only turn is valid.
if len(value) == 0:
return True
# No dict blocks (e.g. [None], [' ']) -> malformed/empty.
dict_blocks = [item for item in value if isinstance(item, dict)]
if not dict_blocks:
return True
non_text_blocks = [item for item in dict_blocks if item.get("type") != "text"]
if non_text_blocks:
return False
text_values = [item.get("text") for item in dict_blocks if item.get("type") == "text"]
if text_values and all(
t is None
or (
isinstance(t, str) and not t.strip().strip("\ufeff\u200b\u200c\u200d\u2060").strip()
)
for t in text_values
):
return True
return False
def _classify_empty(value) -> str:
"""Return a human-readable label for why this value is considered empty."""
if value is None:
return "None"
if isinstance(value, str):
if len(value) == 0:
return "empty_string"
# Only-whitespace or only-invisible (BOM/zero-width) strings render empty.
if not value.strip().strip("\ufeff\u200b\u200c\u200d\u2060").strip():
return "whitespace_only"
if isinstance(value, list):
# Mirrors the VLM/OpenAI content-block handling in is_none_or_empty.
if len(value) == 0:
return "empty_list"
return "empty_vlm_content"
return "valid" # unreachable if is_none_or_empty was True
# Alpaca detection
def find_none_alpaca(dataset: Dataset) -> dict:
"""Scan alpaca dataset for None/empty instruction or output; returns stats with a 'findings' list."""
stats = {
"total_rows": len(dataset),
"none_instruction": 0,
"none_output": 0,
"bad_row_indices": [],
"findings": [],
}
for i, row in enumerate(dataset):
bad = False
for field in ("instruction", "output"):
val = row.get(field)
if is_none_or_empty(val):
stats[f"none_{field}"] = stats.get(f"none_{field}", 0) + 1
bad = True
stats["findings"].append(
{
"row_index": i,
"field": field,
"value_type": _classify_empty(val),
"raw_value": repr(val),
}
)
if bad:
stats["bad_row_indices"].append(i)
return stats
# ChatML / conversational detection
def find_none_chatml(dataset: Dataset, col: str = None) -> dict:
"""Auto-detects the conversation column when col is None; findings hold one entry per bad turn."""
if col is None:
# Reuse _probe_conversation so the all_corrupt path is handled here too.
_cinfo = _probe_conversation(dataset)
if _cinfo is not None:
col = _cinfo["column"]
if col is None or col not in dataset.column_names:
raise ValueError(
f"No conversation column found. "
f"Expected one of {CONVERSATION_COLUMNS}, got columns: {dataset.column_names}"
)
stats = {
"total_rows": len(dataset),
"column": col,
"rows_with_none_turns": 0,
"total_none_turns": 0,
"none_by_role": {},
"none_by_type": {},
"rows_all_none": 0,
"bad_row_indices": [],
"findings": [],
}
for i, row in enumerate(dataset):
conversation = row[col]
if not isinstance(conversation, list):
# Non-list conversation: unusable for training, flag as bad.
vtype = "None" if conversation is None else "invalid_type"
stats["bad_row_indices"].append(i)
stats["rows_with_none_turns"] += 1
stats["total_none_turns"] += 1
stats["rows_all_none"] += 1
stats["none_by_role"]["unknown"] = stats["none_by_role"].get("unknown", 0) + 1
stats["none_by_type"][vtype] = stats["none_by_type"].get(vtype, 0) + 1
stats["findings"].append(
{
"row_index": i,
"turn_index": 0,
"role": "unknown",
"value_type": vtype,
"raw_value": repr(conversation),
}
)
continue
if len(conversation) == 0:
# Zero-turn conversation: flag so it doesn't scan as clean.
stats["bad_row_indices"].append(i)
stats["rows_with_none_turns"] += 1
stats["total_none_turns"] += 1
stats["rows_all_none"] += 1
stats["none_by_role"]["unknown"] = stats["none_by_role"].get("unknown", 0) + 1
stats["none_by_type"]["empty_conversation"] = (
stats["none_by_type"].get("empty_conversation", 0) + 1
)
stats["findings"].append(
{
"row_index": i,
"turn_index": 0,
"role": "unknown",
"value_type": "empty_conversation",
"raw_value": "[]",
}
)
continue
row_findings = []
for turn_idx, turn in enumerate(conversation):
# Non-dict turn - record it rather than crash or silently skip.
if not isinstance(turn, dict):
row_findings.append(
{
"row_index": i,
"turn_index": turn_idx,
"role": "unknown",
"value_type": "None" if turn is None else "invalid_type",
"raw_value": repr(turn),
}
)
stats["none_by_role"]["unknown"] = stats["none_by_role"].get("unknown", 0) + 1
vtype = "None" if turn is None else "invalid_type"
stats["none_by_type"][vtype] = stats["none_by_type"].get(vtype, 0) + 1
continue
# Explicit None check keeps falsy roles (0, "", False) instead of collapsing them to unknown.
r = turn.get("role")
if r is None:
r = turn.get("from")
if r is None:
role = "unknown"
elif isinstance(r, str):
role = r
else:
role = str(r)
# from+value reads value (ShareGPT, even with role); role reads content; from-only reads value.
if "from" in turn and "value" in turn:
content = turn.get("value")
elif "role" in turn:
content = turn.get("content") if "content" in turn else turn.get("value")
elif "from" in turn:
content = turn.get("value")
else:
content = turn.get("content") if "content" in turn else turn.get("value")
# Assistant tool-call turns carry empty content and are valid.
if is_none_or_empty(content) and not (role == "assistant" and turn.get("tool_calls")):
vtype = _classify_empty(content)
row_findings.append(
{
"row_index": i,
"turn_index": turn_idx,
"role": role,
"value_type": vtype,
"raw_value": repr(content),
}
)
stats["none_by_role"][role] = stats["none_by_role"].get(role, 0) + 1
stats["none_by_type"][vtype] = stats["none_by_type"].get(vtype, 0) + 1
if row_findings:
stats["rows_with_none_turns"] += 1
stats["total_none_turns"] += len(row_findings)
stats["bad_row_indices"].append(i)
stats["findings"].extend(row_findings)
if len(row_findings) == len(conversation):
stats["rows_all_none"] += 1
return stats
# Convenience wrappers per format (all delegate to the same scan logic)
def find_none_sharegpt(dataset: Dataset, col: str = None) -> dict:
"""ShareGPT uses 'from'/'value' keys - same scan logic handles both."""
if col is None:
# Probe only 'conversations' for ShareGPT: a corrupt one is still scanned, not swapped for 'messages'.
conv_info = _probe_conversation(dataset, candidates = ("conversations",))
if conv_info is None:
raise ValueError(
f"No valid conversation column found in {dataset.column_names}. "
"Expected a 'conversations' column with 'from'/'value' or 'role'/'content' turn keys."
)
col = conv_info["column"]
return find_none_chatml(dataset, col = col)
def find_none_gptoss(dataset: Dataset, col: str = None) -> dict:
"""gptoss: role/content plus optional thinking/tool_calls. Only content checked."""
if col is None:
# gptoss lives in 'messages': target it whenever present
if "messages" in dataset.column_names:
conv_info = _probe_conversation(dataset, candidates = ("messages",))
else:
conv_info = _probe_conversation(dataset, candidates = ("conversations",))
if conv_info is None:
raise ValueError(
f"No valid conversation column found in {dataset.column_names}. "
"Expected a 'messages' or 'conversations' column with 'role'/'content' turn keys."
)
col = conv_info["column"]
return find_none_chatml(dataset, col = col)
# First match wins: gptoss must come before chatml, since gptoss is chatml with a developer role.
FORMAT_REGISTRY = [
{
"name": "alpaca",
# Alpaca needs no usable chat column (none, or only fully corrupt); a healthy one falls through.
"match": lambda ds, conv: (
{"instruction", "output"}.issubset(ds.column_names)
and (conv is None or conv.get("all_corrupt"))
),
"scan": find_none_alpaca,
},
{
"name": "gptoss",
"match": lambda ds, conv: (
conv is not None
and {"role", "content"} <= conv["turn_keys"]
and "developer" in conv["roles"]
),
"scan": find_none_gptoss,
},
{
"name": "sharegpt",
"match": lambda ds, conv: (conv is not None and {"from", "value"} <= conv["turn_keys"]),
"scan": find_none_sharegpt,
},
{
"name": "chatml",
"match": lambda ds, conv: (
conv is not None
and (
{"role", "content"} <= conv["turn_keys"]
# all_corrupt (every row malformed) also needs has_plausible_turns, so scalar columns are not chatml.
or (conv.get("all_corrupt") and conv.get("has_plausible_turns"))
)
),
"scan": find_none_chatml,
},
]
# Derived list of known format names (used by CLI --format choices).
FORMAT_NAMES = [entry["name"] for entry in FORMAT_REGISTRY]
# Documented aliases accepted by both the Python API and the CLI.
FORMAT_ALIASES = {"gpt-oss": "gptoss"}
def detect_format(dataset: Dataset) -> str:
"""Returns the name of the first FORMAT_REGISTRY entry that matches, or 'unknown'."""
conv_info = _probe_conversation(dataset)
for entry in FORMAT_REGISTRY:
if entry["match"](dataset, conv_info):
return entry["name"]
return "unknown"
def get_scanner(fmt: str):
"""Return the scanner function for a format name, or None if unknown."""
for entry in FORMAT_REGISTRY:
if entry["name"] == fmt:
return entry["scan"]
return None
def scan_dataset(dataset: Dataset, fmt: str = "auto") -> dict:
"""Raises ValueError for an unknown or unsupported format; DatasetDict and IterableDatasetDict are rejected since their split map reads as unknown."""
_dict_types = []
try:
from datasets import DatasetDict as _DatasetDict
_dict_types.append(_DatasetDict)
except ImportError:
pass
try:
from datasets import IterableDatasetDict as _IterableDatasetDict
_dict_types.append(_IterableDatasetDict)
except ImportError:
pass
if _dict_types and isinstance(dataset, tuple(_dict_types)):
raise ValueError(
"scan_dataset requires a single Dataset split, not a DatasetDict. "
f"Available splits: {list(dataset.keys())}. "
"Pass dataset[<split>] or use load_dataset(..., split='train')."
)
# Streaming IterableDataset lacks len()/column_names; fail clearly, not with a downstream TypeError.
try:
from datasets import IterableDataset as _IterableDataset
if isinstance(dataset, _IterableDataset):
raise ValueError(
"scan_dataset requires a materialized Dataset, not an IterableDataset. "
"Load without streaming=True, or materialize a slice first: "
"Dataset.from_list(list(dataset.take(N)))."
)
except ImportError:
pass
fmt = FORMAT_ALIASES.get(fmt, fmt)
was_auto = fmt == "auto"
# Zero-row dataset: return a trivially clean stats dict.
if was_auto and len(dataset) == 0:
return {
"format": "unknown",
"total_rows": 0,
"findings": [],
"bad_row_indices": [],
}
# Always probe so detection and column selection share one scan pass.
conv_info = _probe_conversation(dataset)
if was_auto:
fmt = "unknown"
for entry in FORMAT_REGISTRY:
if entry["match"](dataset, conv_info):
fmt = entry["name"]
break
# No format matched: return stats with format 'unknown' instead of raising, so callers can branch.
if fmt == "unknown":
return {
"format": "unknown",
"total_rows": len(dataset),
"findings": [],
"bad_row_indices": [],
}
scanner = get_scanner(fmt)
if scanner is None:
raise ValueError(f"Unknown or unsupported format: '{fmt}'")
# Column forwarding: on auto-detect pass the probed column
use_probed_col = conv_info is not None and fmt != "alpaca" and was_auto
if use_probed_col:
stats = scanner(dataset, col = conv_info["column"])
else:
stats = scanner(dataset)
stats["format"] = fmt
return stats
# Report printing
def _print_summary_header(stats: dict, fmt: str) -> bool:
"""Print the top-level stats block (shared by all report modes). Returns True if findings exist."""
total = stats["total_rows"]
findings = stats.get("findings", [])
print(f"\n{'=' * 64}")
print(f" None / Empty Detection Report")
print(f"{'=' * 64}")
print(f" Format: {fmt}")
print(f" Total rows: {total}")
if not findings:
if fmt == "unknown":
print(f" Result: NOT SCANNED -- format could not be detected")
else:
print(f" Result: CLEAN -- no None or empty values found")
print(f"{'=' * 64}")
return False
if fmt == "alpaca":
bad_rows = len(stats.get("bad_row_indices", []))
print(f" Rows with Nones: {bad_rows} / {total}")
print(f" None instruction: {stats.get('none_instruction', 0)}")
print(f" None output: {stats.get('none_output', 0)}")
else:
col = stats.get("column", "?")
print(f" Column: {col}")
print(f" Rows with bad turns: {stats['rows_with_none_turns']} / {total}")
print(f" Total bad turns: {len(findings)}")
print(f" By type: {stats.get('none_by_type', {})}")
print(f" By role: {stats.get('none_by_role', {})}")
rows_all = stats.get("rows_all_none", 0)
if rows_all:
print(f" Rows ALL bad: {rows_all} (every turn is None/empty)")
# Count directly; a full set of good indices OOMs on 10M+ row datasets.
bad_indices = set(stats.get("bad_row_indices", []))
clean_count = total - len(bad_indices)
if 0 < clean_count <= 20:
clean_indices = [i for i in range(total) if i not in bad_indices]
print(f" Rows with no Nones: {clean_count} / {total} {clean_indices}")
else:
print(f" Rows with no Nones: {clean_count} / {total}")
print(f"{'=' * 64}")
return True
def print_report(
stats: dict,
fmt: str,
summary_only: bool = False,
):
"""Print a human-readable summary, optionally with full findings list."""
has_findings = _print_summary_header(stats, fmt)
if not has_findings or summary_only:
return
findings = stats.get("findings", [])
print(f"\n {'-' * 60}")
print(f" Findings ({len(findings)} total):")
print(f" {'-' * 60}")
for f in findings:
if fmt == "alpaca":
print(
f" row {f['row_index']:>5d} "
f"field={f['field']:<12s} "
f"type={f['value_type']:<16s} "
f"raw={f['raw_value']}"
)
else:
print(
f" row {f['row_index']:>5d} "
f"turn {f['turn_index']} "
f"role={str(f['role']):<12s} "
f"type={f['value_type']:<16s} "
f"raw={f['raw_value']}"
)
print(f"{'=' * 64}")
def show_row(
dataset: Dataset,
row_indices: list[int],
fmt: str,
col: str = None,
):
"""Not part of the production API; used by test_codex_fixes.py to check row rendering."""
if col is None:
for candidate in ("messages", "conversations", "texts"):
if candidate in dataset.column_names:
col = candidate
break
for ri in row_indices:
if ri < 0 or ri >= len(dataset):
print(f"\n [ERROR] Row {ri} out of range (0-{len(dataset)-1})")
continue
row = dataset[ri]
print(f"\n{'=' * 64}")
print(f" Row {ri}")
print(f"{'=' * 64}")
# Print non-conversation columns; alpaca fields are skipped since the alpaca block prints them.
_ALPACA_FIELDS = {"instruction", "input", "output"}
for key in dataset.column_names:
if key == col:
continue
if fmt == "alpaca" and key in _ALPACA_FIELDS:
continue
val = row[key]
if isinstance(val, str) and len(val) > 120:
val = val[:120] + "..."
print(f" {key}: {val}")
if fmt == "alpaca":
for field in ("instruction", "input", "output"):
val = row.get(field)
status = " [NONE]" if is_none_or_empty(val) else ""
if val and len(str(val)) > 200:
val = str(val)[:200] + "..."
print(f" {field}: {val}{status}")
elif col:
conversation = row[col]
if isinstance(conversation, list):
def _is_bad_turn(t):
if not isinstance(t, dict):
return True
# Mirror scanner logic: from+value wins, then role, then from alone.
if "from" in t and "value" in t:
c = t.get("value")
elif "role" in t:
c = t.get("content") if "content" in t else t.get("value")
elif "from" in t:
c = t.get("value")
else:
c = t.get("content") if "content" in t else t.get("value")
# Mirrors the scanner: only assistant turns may skip empty content when they carry tool_calls.
r = t.get("role") if t.get("role") is not None else t.get("from")
if is_none_or_empty(c) and not (str(r) == "assistant" and t.get("tool_calls")):
return True
return False
none_count = sum(1 for t in conversation if _is_bad_turn(t))
print(f" {col}: {len(conversation)} turns ({none_count} None)")
print(f" {'-' * 60}")
for i, turn in enumerate(conversation):
if not isinstance(turn, dict):
label = "None" if turn is None else "invalid_type"
print(f" [{i:>3d}] {'unknown':<12s} [{label}] << NONE")
continue
r = turn.get("role")
if r is None:
r = turn.get("from")
role = "?" if r is None else str(r)
# Mirror scanner logic: from+value wins, then role, then from alone.
if "from" in turn and "value" in turn:
content = turn.get("value")
elif "role" in turn:
content = turn.get("content") if "content" in turn else turn.get("value")
elif "from" in turn:
content = turn.get("value")
else:
content = turn.get("content") if "content" in turn else turn.get("value")
if is_none_or_empty(content) and not (
role == "assistant" and turn.get("tool_calls")
):
status = " << NONE"
else:
status = ""
if content is None:
preview = "None"
else:
preview_str = str(content)
if len(preview_str) > 150:
preview = preview_str[:150].replace("\n", "\\n") + "..."
else:
preview = preview_str.replace("\n", "\\n")
print(f" [{i:>3d}] {role:<12s} {preview}{status}")
print(f"{'=' * 64}")
# CLI entry point
if __name__ == "__main__":
import argparse
import os
import sys
parser = argparse.ArgumentParser(
prog = "dataset_none_detect",
description = "Scan a HuggingFace dataset for None/empty content turns.",
formatter_class = argparse.RawDescriptionHelpFormatter,
epilog = """
examples:
python dataset_none_detect.py org/my-dataset
python dataset_none_detect.py org/my-dataset --split train
python dataset_none_detect.py org/my-dataset --format sharegpt
python dataset_none_detect.py org/my-dataset --summary-only
python dataset_none_detect.py org/my-dataset --token hf_...
""",
)
parser.add_argument("dataset", help = "HuggingFace dataset repo id (e.g. org/my-dataset)")
parser.add_argument("--split", default = "train", help = "Dataset split to load (default: train)")
parser.add_argument(
"--format",
default = "auto",
choices = ["auto"] + FORMAT_NAMES + list(FORMAT_ALIASES),
help = "Force a specific format instead of auto-detecting (default: auto). "
"Documented aliases (e.g. 'gpt-oss' for 'gptoss') are also accepted.",
)
parser.add_argument(
"--summary-only",
action = "store_true",
help = "Print summary header only - skip the per-turn findings list",
)
parser.add_argument(
"--token",
default = os.environ.get("HF_TOKEN"),
help = (
"HuggingFace API token for private datasets (default: $HF_TOKEN). "
"Prefer setting $HF_TOKEN; passing --token on the command line "
"exposes it in process listings."
),
)
args = parser.parse_args()
try:
from datasets import load_dataset
except ImportError:
print(
"Error: 'datasets' package not found. Install with: pip install datasets",
file = sys.stderr,
)
sys.exit(1)
print(f"Loading {args.dataset!r} (split={args.split!r})...")
try:
ds = load_dataset(args.dataset, split = args.split, token = args.token)
except Exception as exc:
# Some library versions include the Authorization header in errors; redact it.
msg = str(exc)
if args.token:
msg = msg.replace(args.token, "hf_***REDACTED***")
print(f"Error loading dataset: {msg}", file = sys.stderr)
sys.exit(1)
print(f"Loaded {len(ds)} rows, columns: {ds.column_names}")
try:
stats = scan_dataset(ds, fmt = args.format)
except ValueError as exc:
print(f"Error: {exc}", file = sys.stderr)
sys.exit(1)
print_report(stats, stats["format"], summary_only = args.summary_only)