Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Collapse blank line runs with a literal-led pattern (about 10x faster) (#15950)

Daniel Han761487fa08ae
diff --git a/studio/backend/core/external_import/__init__.py b/studio/backend/core/external_import/__init__.py
index 5aff284fe61..cf5ede7250b 100644
--- a/studio/backend/core/external_import/__init__.py
+++ b/studio/backend/core/external_import/__init__.py
@@ -19,6 +19,8 @@ from typing import Callable, Iterator, Optional
from loggers import get_logger
from storage import studio_db
+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")
+
logger = get_logger(__name__)
TITLE_MAX_CHARS = 120
@@ -170,7 +172,7 @@ def iso_ms(value) -> Optional[int]:
def clean_text(text: str) -> str:
- return re.sub(r"\n{3,}", "\n\n", text).strip()
+ return _BLANK_LINE_RUN.sub("\n\n", text).strip()
def tool_call(tool_id: str, block: dict) -> dict:
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py
index 5621fd9bf2c..621db3f6296 100644
--- a/studio/backend/core/inference/_html_to_md.py
+++ b/studio/backend/core/inference/_html_to_md.py
@@ -334,6 +334,7 @@ _MAX_REPEATED_CELL_CHARS = 200
_MIN_SCOPE_SPAN_CHARS = 256
_INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}
+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")
# citation lists after words use commas or en dashes; hyphens, minuses, and thousands stay exponents
_CITATION_LIST = re.compile(r"\d+(?:\s*[–,]\s*\d+)+")
_THOUSANDS = re.compile(r"\d{1,3}(?:,\d{3})+")
@@ -1068,7 +1069,7 @@ class _MarkdownRenderer(HTMLParser):
def _prefix_blockquote(self, content: str) -> str:
"""Prefix every line of *content* with ``> ``."""
content = re.sub(r"[ \t]+$", "", content, flags = re.MULTILINE)
- content = re.sub(r"\n{3,}", "\n\n", content).strip()
+ content = _BLANK_LINE_RUN.sub("\n\n", content).strip()
if not content:
return ""
lines = content.split("\n")
@@ -2158,7 +2159,7 @@ def _strip_boilerplate_lines(text: str, site_links: SiteLinks | None = None) ->
continue
out.append(line)
# Collapse blank runs the dropped lines may have left behind.
- return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()
+ return _BLANK_LINE_RUN.sub("\n\n", "\n".join(out)).strip()
class _TokenLog:
diff --git a/studio/backend/core/rag/office_formats.py b/studio/backend/core/rag/office_formats.py
index 7cda4c84be7..576aebad406 100644
--- a/studio/backend/core/rag/office_formats.py
+++ b/studio/backend/core/rag/office_formats.py
@@ -210,6 +210,7 @@ _BUILTIN_DATE_FORMATS = {
_BUILTIN_DURATION_FORMATS = {46} # [h]:mm:ss
+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")
# Classified as openpyxl does: quoted text and locale tags aside, any date or time token.
_FORMAT_STRIP = re.compile(r'".*?"|\[(?!hh?\]|mm?\]|ss?\])[^\]]*\]')
_FORMAT_DURATION = re.compile(
@@ -1231,7 +1232,7 @@ def _word_text(raw: str) -> str:
# A cell ends in 0x07; a row ends in one more.
text = "".join(out).replace("\x07\x07", "\n").replace("\x07", " | ")
text = text.translate({0x0D: "\n", 0x0B: "\n", 0x0C: "\n", 0x1E: "-", 0x1F: None})
- text = re.sub(r"\n{3,}", "\n\n", text)
+ text = _BLANK_LINE_RUN.sub("\n\n", text)
return _clean_control(text).strip()
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 50d4bc855be..c9c4085d09e 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -210,7 +210,7 @@ def _auto_scrape_default() -> int:
_MD_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")
_PERCENT_ESCAPE = re.compile(r"%[0-9A-Fa-f]{2}")
_LIST_PREFIX = re.compile(r"^(?:[\*\-\+•]|\d+[.)])\s")
-_BLANK_RUN = re.compile(r"\n{3,}")
+_BLANK_RUN = re.compile(r"\n\n\n+")
# Tracking URLs are single 80+ char tokens that small models echo.
_LONG_TOKEN = re.compile(r"\S{80,}")
diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
index cd794eddb0a..fb91585924a 100644
--- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
+++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
@@ -17,6 +17,8 @@ MAX_CHUNK_SIZE = 20000
_MIN_BREAK_RATIO = 0.6
_CACHE_DIR = LazyPath(unstructured_seed_cache_root)
_WHITESPACE = re.compile(r"\s+")
+# Same matches as \n{3,}, but the literal prefix lets re skip to each newline instead of trying every offset.
+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")
# Bump when chunk output changes so cached Parquet seeds are rebuilt.
_CHUNKER_VERSION = 2
@@ -252,7 +254,7 @@ def load_unstructured_text_file(path: Path) -> str:
def normalize_unstructured_text(text: str) -> str:
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
- return re.sub(r"\n{3,}", "\n\n", normalized).strip()
+ return _BLANK_LINE_RUN.sub("\n\n", normalized).strip()
def split_text_into_chunks(*, text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
diff --git a/studio/backend/tests/test_unstructured_seed_blank_lines.py b/studio/backend/tests/test_unstructured_seed_blank_lines.py
new file mode 100644
index 00000000000..5f99357a28f
--- /dev/null
+++ b/studio/backend/tests/test_unstructured_seed_blank_lines.py
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import importlib.util
+import random
+import re
+from pathlib import Path
+
+_CHUNKING = (
+ Path(__file__).resolve().parent.parent
+ / "plugins"
+ / "data-designer-unstructured-seed"
+ / "src"
+ / "data_designer_unstructured_seed"
+ / "chunking.py"
+)
+
+
+def _load_chunking():
+ spec = importlib.util.spec_from_file_location("_uss_blank_lines_under_test", _CHUNKING)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+chunking = _load_chunking()
+
+
+def _previous_normalize(text):
+ normalized = text.replace("\r\n", "\n").replace("\r", "\n")
+ return re.sub(r"\n{3,}", "\n\n", normalized).strip()
+
+
+def test_normalize_matches_previous_output():
+ rng = random.Random(0)
+ alphabet = ["\n", "\n", "\r", "\r\n", "a", " ", "\t", "\x85", "
", "é", "\n\n\n"]
+ cases = ["", "\n", "\n\n", "\n\n\n", "a\n\n\n\n\nb", "\r\r\r", " \n\n\n ", "x" * 50]
+ cases += [
+ "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 60))) for _ in range(20000)
+ ]
+ old = re.compile(r"\n{3,}")
+ for text in cases:
+ assert chunking.normalize_unstructured_text(text) == _previous_normalize(text), repr(text)
+ assert [m.span() for m in chunking._BLANK_LINE_RUN.finditer(text)] == [
+ m.span() for m in old.finditer(text)
+ ], repr(text)
diff --git a/unsloth/dataprep/ebooks.py b/unsloth/dataprep/ebooks.py
index e4ea3b73d08..412f670e39f 100644
--- a/unsloth/dataprep/ebooks.py
+++ b/unsloth/dataprep/ebooks.py
@@ -18,6 +18,8 @@ from urllib.parse import unquote, urlsplit
from xml.etree import ElementTree as ET
from zipfile import BadZipFile, ZipFile
+_BLANK_LINE_RUN = re.compile(r"\n\n\n+")
+
__all__ = ["load_ebook_dataset"]
_MAX_MEMBER_BYTES = 32 * 1024 * 1024
@@ -160,7 +162,7 @@ def _html_text(data):
text = unicodedata.normalize("NFC", "".join(parser.parts))
text = re.sub(r"[^\S\n]+", " ", text)
text = re.sub(r" *\n *", "\n", text)
- text = re.sub(r"\n{3,}", "\n\n", text).strip()
+ text = _BLANK_LINE_RUN.sub("\n\n", text).strip()
return re.sub(
"\ue000(\\d+)\ue001",
lambda m: unicodedata.normalize("NFC", parser.preformatted[int(m[1])]),
diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py
index fb51243d81b..fadc923ada0 100644
--- a/unsloth/dataprep/raw_text.py
+++ b/unsloth/dataprep/raw_text.py
@@ -359,7 +359,7 @@ class TextPreprocessor:
_TEXT_CHARS = _TextCharTable()
_MULTIPLE_SPACES_PATTERN = re.compile(r"[ ]{2,}")
_NEWLINE_SPACES_PATTERN = re.compile(r" *\n *")
- _MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")
+ _MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n\n\n+")
_CHAPTER_PATTERN = re.compile(r"^# (.+)$", re.MULTILINE)
_SECTION_PATTERN = re.compile(r"^## (.+)$", re.MULTILINE)
_SUBSECTION_PATTERN = re.compile(r"^### (.+)$", re.MULTILINE)