Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop clean_text deleting every non-ASCII character (#10741)

Vineeth Sai Varikuntlad49efcfe71bc
diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py
index 385f6960331..c63c0e15bf6 100644
--- a/tests/test_raw_text.py
+++ b/tests/test_raw_text.py
@@ -4,6 +4,7 @@
import sys
import os
import tempfile
+import unicodedata
from pathlib import Path
import importlib.util
@@ -198,16 +199,16 @@ def test_raw_text_loader():
# Spaces around newlines trimmed on both sides, even across multiple newlines.
assert preprocessor.clean_text("foo \n\n bar") == "foo\n\nbar"
- # Stripping a non-ASCII char between spaces must not leave a double space
- assert preprocessor.clean_text("word1 \u00a9 word2") == "word1 word2"
- assert preprocessor.clean_text("a \u00e9 b") == "a b"
- assert preprocessor.clean_text("prefix \U0001f600 suffix") == "prefix suffix"
+ # Stripping an invisible character between spaces must not leave a double space.
+ assert preprocessor.clean_text("word1 \u200b word2") == "word1 word2"
+ assert preprocessor.clean_text("a \ue000 b") == "a b"
+ assert preprocessor.clean_text("prefix \ufffd suffix") == "prefix suffix"
- # Stripping a non-ASCII char adjacent to a newline must not leave a stray space.
- assert preprocessor.clean_text("foo \u00e9\nbar") == "foo\nbar"
- assert preprocessor.clean_text("foo\n\u00e9 bar") == "foo\nbar"
- # The double-space collapse must not swallow a paragraph break near a non-ASCII char.
- assert preprocessor.clean_text("a \u00a9\n\nb") == "a\n\nb"
+ # Stripping an invisible character adjacent to a newline must not leave a stray space.
+ assert preprocessor.clean_text("foo \u200b\nbar") == "foo\nbar"
+ assert preprocessor.clean_text("foo\n\ue000 bar") == "foo\nbar"
+ # The double-space collapse must not swallow a paragraph break near an invisible character.
+ assert preprocessor.clean_text("a \u200b\n\nb") == "a\n\nb"
# Idempotence: clean_text twice == once.
idempotent_inputs = [
@@ -227,17 +228,93 @@ def test_raw_text_loader():
assert stats["total_samples"] > 0, "Should count samples"
assert "warnings" in stats, "Should include warnings"
- print("✅ All tests passed!")
+ # Plain ASCII: a Windows console is cp1252 and cannot encode a check mark, so one
+ # here killed the driver mid-file. pytest hid it, capturing stdout as UTF-8.
+ print("All tests passed!")
return True
- except Exception as e:
- print(f"❌ Test failed: {e}")
- return False
-
+ # No `except Exception: return False` here: it swallowed the failure and still reported a
+ # pass, so every assertion above ran in CI unable to fail it. That is how this shipped.
finally:
os.unlink(test_file)
+def test_clean_text_keeps_text_in_any_script():
+ """Top level on purpose: an assertion inside test_raw_text_loader used to be swallowed."""
+ preprocessor = TextPreprocessor()
+ for script_text in [
+ "Le caf\u00e9 \u00e9tait tr\u00e8s bon.",
+ "\u00bfD\u00f3nde est\u00e1 la ni\u00f1a?",
+ "Gr\u00f6\u00dfe und Stra\u00dfe",
+ "\u673a\u5668\u5b66\u4e60\u5f88\u6709\u8da3\u3002",
+ "\u3053\u3093\u306b\u3061\u306f\u3001\u4e16\u754c\u3002",
+ "\u0645\u0631\u062d\u0628\u0627\u060c \u0628\u0627\u0644\u0639\u0627\u0644\u0645",
+ "\u041f\u0440\u0438\u0432\u0435\u0442, \u043c\u0438\u0440!",
+ "\u0928\u092e\u0938\u094d\u0924\u0947 \u0926\u0941\u0928\u093f\u092f\u093e\u0964",
+ "\uc548\ub155\ud558\uc138\uc694.",
+ "\u0393\u03b5\u03b9\u03ac \u03c3\u03bf\u03c5",
+ "\u0645\u06cc\u200c\u062e\u0648\u0627\u0647\u0645",
+ "\u0dc1\u0dca\u200d\u0dbb\u0dd3",
+ "Price: \u20ac100, x \u2264 4, \u00a9 2026 Acme\u2122",
+ "I \u2764\ufe0f you 1\ufe0f\u20e3 \U0001f468\u200d\U0001f469\u200d\U0001f467",
+ "\U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f",
+ "\u06dd\u0661\u0662 \u0600\u0663",
+ "\U00013000\U00013430\U00013001",
+ "f\u2061(x) = a\u2062b",
+ # Garay (Unicode 16) is unassigned in older interpreters' databases and must survive anyway.
+ "\U00010d50\U00010d51",
+ ]:
+ assert (
+ preprocessor.clean_text(script_text) == script_text
+ ), f"clean_text must keep text in any script: {script_text!r}"
+
+
+def test_clean_text_drops_invisible_characters():
+ """The control for the test above: invisible characters are still removed."""
+ preprocessor = TextPreprocessor()
+ for raw, expected in [
+ ("a\x00b", "ab"),
+ ("a\x1bb", "ab"),
+ ("\ufeffhello", "hello"),
+ ("co\u00adop", "coop"),
+ ("a\u200bb", "ab"),
+ ("\u202eabc", "abc"),
+ ("a\u200eb\u2060c", "abc"),
+ ("\u2066abc\u2069", "abc"),
+ ("a\ue000b", "ab"),
+ ("a\ufffdb", "ab"),
+ ("a\uffffb", "ab"),
+ ]:
+ assert preprocessor.clean_text(raw) == expected, raw
+
+
+def test_clean_text_decision_cannot_drift_between_interpreters():
+ """unicodedata ships with the interpreter (3.9 has Unicode 13.0, 3.14 has 16.0), and
+ Unicode freezes only Cc, Co and Cs. Keying on any other category would clean the same
+ corpus differently per Python. https://www.unicode.org/policies/property_value_stability_table.html
+ """
+ immutable = {"Cc", "Co", "Cs"}
+ assert set(raw_text_module._TextCharTable._DROP_CATEGORIES) <= immutable, (
+ "clean_text may only key on the General_Category values Unicode has frozen "
+ f"({sorted(immutable)}); the rest differ between Python versions."
+ )
+
+ # Everything else dropped is named explicitly, not derived from the database.
+ preprocessor = TextPreprocessor()
+ for codepoint in (0x00AD, 0x200B, 0x200E, 0x2060, 0x2066, 0xFEFF, 0xFFFD, 0xE0001):
+ assert preprocessor.clean_text(f"a{chr(codepoint)}b") == "ab", hex(codepoint)
+
+ # Unassigned (Cn) means newer than this interpreter's database, so it must survive.
+ unassigned = [
+ cp
+ for cp in range(0x10D40, 0x10D90)
+ if unicodedata.category(chr(cp)) == "Cn" and not (cp & 0xFFFE) == 0xFFFE
+ ]
+ for codepoint in unassigned[:8]:
+ text = f"a{chr(codepoint)}b"
+ assert preprocessor.clean_text(text) == text, hex(codepoint)
+
+
def test_smart_chunk_text_single_chunk_no_eos_returns_plain_list():
"""smart_chunk_text's single-chunk branch must return a plain list for
input_ids even when the tokenizer has no eos_token_id, matching the
@@ -288,7 +365,7 @@ def test_smart_chunk_text_single_chunk_no_eos_returns_plain_list():
input_ids, list
), f"input_ids should be a plain list even without an eos_token_id, got {type(input_ids)}"
assert input_ids == [0, 1, 2, 3], f"unexpected input_ids: {input_ids}"
- print("✅ test_smart_chunk_text_single_chunk_no_eos_returns_plain_list passed!")
+ print("test_smart_chunk_text_single_chunk_no_eos_returns_plain_list passed")
return True
@@ -344,7 +421,7 @@ def test_smart_chunk_text_no_eos_on_intermediate_full_chunks():
chunk.endswith("</s>") == is_last
), f"chunk {i} (last={is_last}) eos suffix mismatch: {chunk!r}"
- print("✅ test_smart_chunk_text_no_eos_on_intermediate_full_chunks passed!")
+ print("test_smart_chunk_text_no_eos_on_intermediate_full_chunks passed")
return True
@@ -725,6 +802,9 @@ def test_validate_dataset_reports_zero_min_length_when_nothing_has_content():
if __name__ == "__main__":
success = test_raw_text_loader()
+ test_clean_text_keeps_text_in_any_script()
+ test_clean_text_drops_invisible_characters()
+ test_clean_text_decision_cannot_drift_between_interpreters()
success = test_smart_chunk_text_single_chunk_no_eos_returns_plain_list() and success
success = test_smart_chunk_text_no_eos_on_intermediate_full_chunks() and success
success = test_load_from_file_skips_non_object_json_lines() and success
diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py
index 8e744da7229..9a06ad09da7 100644
--- a/unsloth/dataprep/raw_text.py
+++ b/unsloth/dataprep/raw_text.py
@@ -13,6 +13,7 @@ import os
import re
import json
import csv
+import unicodedata
from typing import List, Dict, Any, Union, Optional
from datasets import Dataset
from pathlib import Path
@@ -300,9 +301,54 @@ def _iter_column(dataset, column):
yield from dataset[column]
+class _TextCharTable(dict):
+ """str.translate table for clean_text, filled in the first time each character is seen."""
+
+ # Drop only the invisible: control, private-use and surrogate code points, noncharacters, and the
+ # format characters below. Every other format character (ZWJ, ZWNJ, ayah signs, emoji tags) is text,
+ # and Cn just means newer than this interpreter's Unicode database.
+ _DROP_CATEGORIES = frozenset(("Cc", "Co", "Cs"))
+ _JUNK_CHARS = frozenset(
+ chr(codepoint)
+ for first, last in (
+ (0x00AD, 0x00AD), # soft hyphen
+ (0x061C, 0x061C), # Arabic letter mark
+ (0x200B, 0x200B), # zero-width space
+ (0x200E, 0x200F), # left-to-right and right-to-left marks
+ (0x202A, 0x202E), # bidi embeddings and overrides
+ (0x2060, 0x2060), # word joiner
+ (0x2066, 0x206F), # bidi isolates and deprecated format controls
+ (0xFEFF, 0xFEFF), # byte order mark
+ (0xFFF9, 0xFFFB), # interlinear annotation
+ (0xFFFD, 0xFFFD), # replacement character left by a bad decode
+ (0xE0001, 0xE0001), # deprecated language tag
+ )
+ for codepoint in range(first, last + 1)
+ )
+
+ def __missing__(self, codepoint):
+ char = chr(codepoint)
+ if char == "\n":
+ keep = True
+ elif (
+ char in self._JUNK_CHARS
+ or 0xFDD0 <= codepoint <= 0xFDEF
+ or (codepoint & 0xFFFE) == 0xFFFE
+ ):
+ keep = False
+ else:
+ keep = unicodedata.category(char) not in self._DROP_CATEGORIES
+ self[codepoint] = codepoint if keep else None
+ return self[codepoint]
+
+ def __reduce__(self):
+ # Pickle empty: the MLX path pickles TextPreprocessor by value, and datasets would hash the cache.
+ return type(self), ()
+
+
class TextPreprocessor:
_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")
- _INVALID_CHARS_PATTERN = re.compile(r"[^\x20-\x7E\n]")
+ _TEXT_CHARS = _TextCharTable()
_MULTIPLE_SPACES_PATTERN = re.compile(r"[ ]{2,}")
_NEWLINE_SPACES_PATTERN = re.compile(r" *\n *")
_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")
@@ -315,7 +361,7 @@ class TextPreprocessor:
"""Remove unwanted characters, normalize whitespace"""
text = text.replace("\r\n", "\n").replace("\r", "\n")
text = self._WHITESPACE_PATTERN.sub(" ", text)
- text = self._INVALID_CHARS_PATTERN.sub("", text)
+ text = text.translate(self._TEXT_CHARS)
text = self._MULTIPLE_SPACES_PATTERN.sub(" ", text)
text = self._NEWLINE_SPACES_PATTERN.sub("\n", text)
text = self._MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text)