Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Collapse blank line runs with a literal-led pattern (about 10x faster) (#15950)
Daniel Han761487fa08ae
diff --git a/studio/backend/core/external_import/__init__.py b/studio/backend/core/external_import/__init__.pyindex 5aff284fe61..cf5ede7250b 100644--- a/studio/backend/core/external_import/__init__.py+++ b/studio/backend/core/external_import/__init__.py@@ -19,6 +19,8 @@ from typing import Callable, Iterator, Optionalfrom loggers import get_loggerfrom storage import studio_db+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")+logger = get_logger(__name__)TITLE_MAX_CHARS = 120@@ -170,7 +172,7 @@ def iso_ms(value) -> Optional[int]:def clean_text(text: str) -> str:- return re.sub(r"\n{3,}", "\n\n", text).strip()+ return _BLANK_LINE_RUN.sub("\n\n", text).strip()def tool_call(tool_id: str, block: dict) -> dict:diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.pyindex 5621fd9bf2c..621db3f6296 100644--- a/studio/backend/core/inference/_html_to_md.py+++ b/studio/backend/core/inference/_html_to_md.py@@ -334,6 +334,7 @@ _MAX_REPEATED_CELL_CHARS = 200_MIN_SCOPE_SPAN_CHARS = 256_INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")# citation lists after words use commas or en dashes; hyphens, minuses, and thousands stay exponents_CITATION_LIST = re.compile(r"\d+(?:\s*[–,]\s*\d+)+")_THOUSANDS = re.compile(r"\d{1,3}(?:,\d{3})+")@@ -1068,7 +1069,7 @@ class _MarkdownRenderer(HTMLParser):def _prefix_blockquote(self, content: str) -> str:"""Prefix every line of *content* with ``> ``."""content = re.sub(r"[ \t]+$", "", content, flags = re.MULTILINE)- content = re.sub(r"\n{3,}", "\n\n", content).strip()+ content = _BLANK_LINE_RUN.sub("\n\n", content).strip()if not content:return ""lines = content.split("\n")@@ -2158,7 +2159,7 @@ def _strip_boilerplate_lines(text: str, site_links: SiteLinks | None = None) ->continueout.append(line)# Collapse blank runs the dropped lines may have left behind.- return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()+ return _BLANK_LINE_RUN.sub("\n\n", "\n".join(out)).strip()class _TokenLog:diff --git a/studio/backend/core/rag/office_formats.py b/studio/backend/core/rag/office_formats.pyindex 7cda4c84be7..576aebad406 100644--- a/studio/backend/core/rag/office_formats.py+++ b/studio/backend/core/rag/office_formats.py@@ -210,6 +210,7 @@ _BUILTIN_DATE_FORMATS = {_BUILTIN_DURATION_FORMATS = {46} # [h]:mm:ss+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")# Classified as openpyxl does: quoted text and locale tags aside, any date or time token._FORMAT_STRIP = re.compile(r'".*?"|\[(?!hh?\]|mm?\]|ss?\])[^\]]*\]')_FORMAT_DURATION = re.compile(@@ -1231,7 +1232,7 @@ def _word_text(raw: str) -> str:# A cell ends in 0x07; a row ends in one more.text = "".join(out).replace("\x07\x07", "\n").replace("\x07", " | ")text = text.translate({0x0D: "\n", 0x0B: "\n", 0x0C: "\n", 0x1E: "-", 0x1F: None})- text = re.sub(r"\n{3,}", "\n\n", text)+ text = _BLANK_LINE_RUN.sub("\n\n", text)return _clean_control(text).strip()diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.pyindex 50d4bc855be..c9c4085d09e 100644--- a/studio/backend/core/research_runs.py+++ b/studio/backend/core/research_runs.py@@ -210,7 +210,7 @@ def _auto_scrape_default() -> int:_MD_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")_PERCENT_ESCAPE = re.compile(r"%[0-9A-Fa-f]{2}")_LIST_PREFIX = re.compile(r"^(?:[\*\-\+•]|\d+[.)])\s")-_BLANK_RUN = re.compile(r"\n{3,}")+_BLANK_RUN = re.compile(r"\n\n\n+")# Tracking URLs are single 80+ char tokens that small models echo._LONG_TOKEN = re.compile(r"\S{80,}")diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.pyindex cd794eddb0a..fb91585924a 100644--- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py+++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py@@ -17,6 +17,8 @@ MAX_CHUNK_SIZE = 20000_MIN_BREAK_RATIO = 0.6_CACHE_DIR = LazyPath(unstructured_seed_cache_root)_WHITESPACE = re.compile(r"\s+")+# Same matches as \n{3,}, but the literal prefix lets re skip to each newline instead of trying every offset.+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")# Bump when chunk output changes so cached Parquet seeds are rebuilt._CHUNKER_VERSION = 2@@ -252,7 +254,7 @@ def load_unstructured_text_file(path: Path) -> str:def normalize_unstructured_text(text: str) -> str:normalized = text.replace("\r\n", "\n").replace("\r", "\n")- return re.sub(r"\n{3,}", "\n\n", normalized).strip()+ return _BLANK_LINE_RUN.sub("\n\n", normalized).strip()def split_text_into_chunks(*, text: str, chunk_size: int, chunk_overlap: int) -> list[str]:diff --git a/studio/backend/tests/test_unstructured_seed_blank_lines.py b/studio/backend/tests/test_unstructured_seed_blank_lines.pynew file mode 100644index 00000000000..5f99357a28f--- /dev/null+++ b/studio/backend/tests/test_unstructured_seed_blank_lines.py@@ -0,0 +1,46 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import importlib.util+import random+import re+from pathlib import Path++_CHUNKING = (+ Path(__file__).resolve().parent.parent+ / "plugins"+ / "data-designer-unstructured-seed"+ / "src"+ / "data_designer_unstructured_seed"+ / "chunking.py"+)+++def _load_chunking():+ spec = importlib.util.spec_from_file_location("_uss_blank_lines_under_test", _CHUNKING)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++chunking = _load_chunking()+++def _previous_normalize(text):+ normalized = text.replace("\r\n", "\n").replace("\r", "\n")+ return re.sub(r"\n{3,}", "\n\n", normalized).strip()+++def test_normalize_matches_previous_output():+ rng = random.Random(0)+ alphabet = ["\n", "\n", "\r", "\r\n", "a", " ", "\t", "\x85", " ", "é", "\n\n\n"]+ cases = ["", "\n", "\n\n", "\n\n\n", "a\n\n\n\n\nb", "\r\r\r", " \n\n\n ", "x" * 50]+ cases += [+ "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 60))) for _ in range(20000)+ ]+ old = re.compile(r"\n{3,}")+ for text in cases:+ assert chunking.normalize_unstructured_text(text) == _previous_normalize(text), repr(text)+ assert [m.span() for m in chunking._BLANK_LINE_RUN.finditer(text)] == [+ m.span() for m in old.finditer(text)+ ], repr(text)diff --git a/unsloth/dataprep/ebooks.py b/unsloth/dataprep/ebooks.pyindex e4ea3b73d08..412f670e39f 100644--- a/unsloth/dataprep/ebooks.py+++ b/unsloth/dataprep/ebooks.py@@ -18,6 +18,8 @@ from urllib.parse import unquote, urlsplitfrom xml.etree import ElementTree as ETfrom zipfile import BadZipFile, ZipFile+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")+__all__ = ["load_ebook_dataset"]_MAX_MEMBER_BYTES = 32 * 1024 * 1024@@ -160,7 +162,7 @@ def _html_text(data):text = unicodedata.normalize("NFC", "".join(parser.parts))text = re.sub(r"[^\S\n]+", " ", text)text = re.sub(r" *\n *", "\n", text)- text = re.sub(r"\n{3,}", "\n\n", text).strip()+ text = _BLANK_LINE_RUN.sub("\n\n", text).strip()return re.sub("\ue000(\\d+)\ue001",lambda m: unicodedata.normalize("NFC", parser.preformatted[int(m[1])]),diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.pyindex fb51243d81b..fadc923ada0 100644--- a/unsloth/dataprep/raw_text.py+++ b/unsloth/dataprep/raw_text.py@@ -359,7 +359,7 @@ class TextPreprocessor:_TEXT_CHARS = _TextCharTable()_MULTIPLE_SPACES_PATTERN = re.compile(r"[ ]{2,}")_NEWLINE_SPACES_PATTERN = re.compile(r" *\n *")- _MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")+ _MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n\n\n+")_CHAPTER_PATTERN = re.compile(r"^# (.+)$", re.MULTILINE)_SECTION_PATTERN = re.compile(r"^## (.+)$", re.MULTILINE)_SUBSECTION_PATTERN = re.compile(r"^### (.+)$", re.MULTILINE)