Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: split seed Hub file names by string and compile split label matchers once (#16352)

Daniel Hane811e74b3d3e
diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py
index bc035f1ef28..f3cd3d6fe72 100644
--- a/studio/backend/routes/data_recipe/seed.py
+++ b/studio/backend/routes/data_recipe/seed.py
@@ -8,6 +8,7 @@ from __future__ import annotations
from core.training.account_jobs import account_path, managed_account
import base64
import binascii
+import functools
import json
import os
import re
@@ -302,9 +303,26 @@ def _class_end(pattern: str, start: int) -> int:
return closing if closing > start else -1
+def _posix_tail(path: str) -> list[str]:
+ """PurePosixPath(path).parts without the root: split on "/", empty and "." parts dropped."""
+ return [part for part in path.split("/") if part and part != "."]
+
+
+def _name_parts(path: str) -> tuple[str, str]:
+ """Path(path).suffix and .stem; by hand only for plain relative ASCII names, which every pathlib flavour and version reads alike."""
+ if path.isascii() and not path.startswith("/") and "\\" not in path and ":" not in path:
+ name = path[path.rfind("/") + 1 :]
+ dot = name.rfind(".")
+ if name[:1] not in ("", ".") and dot < len(name) - 1:
+ return (name[dot:], name[:dot]) if dot > 0 else ("", name)
+ parsed = Path(path)
+ return parsed.suffix, parsed.stem
+
+
def _marked_parts(path: str, prefix: str, which: str) -> list[str]:
- parsed = PurePosixPath(path)
- parts = parsed.parts if which == "parts" else parsed.parent.parts
+ parts = _posix_tail(path)
+ if which != "parts":
+ parts = parts[:-1]
return [part for part in parts if part.startswith(prefix) and set(part) != {"."}]
@@ -338,7 +356,7 @@ def _loader_path_facts(path: str) -> tuple[int, int, str]:
return (
len(_marked_parts(path, ".", "parts")),
len(_marked_parts(path, "__", "parent_parts")),
- PurePosixPath(path).name,
+ (_posix_tail(path) or [""])[-1],
)
@@ -366,7 +384,12 @@ def _common_parent(paths: list[str]) -> str:
def _label_in_name(name: str, label: str) -> bool:
"""Match the label in place, not by tokens, so sample-10BT survives; digits count as separators: train1 is train, training is not."""
- return re.search(rf"(?:^|{_SEP}){re.escape(label)}(?:$|{_SEP})", name) is not None
+ return _label_matcher(label).search(name) is not None
+
+
[email protected]_cache(maxsize = 256)
+def _label_matcher(label: str) -> re.Pattern[str]:
+ return re.compile(rf"(?:^|{_SEP}){re.escape(label)}(?:$|{_SEP})")
def _split_folder(lowered_path: str, labels: tuple[str, ...]) -> bool:
@@ -401,7 +424,7 @@ def _split_rank(path: str, split_lower: str) -> int:
if _split_folder(lowered, labels):
return 0
# Only the final extension comes off: questions.train.parquet keeps its split.
- stem = Path(lowered).stem
+ stem = _name_parts(lowered)[1]
if any(_label_in_name(stem, label) for label in labels):
return 1
return 2
@@ -418,7 +441,7 @@ def _in_subset(data_files: list[str], subset: str | None, split_lower: str) -> l
]
if in_folder:
return in_folder
- named = [f for f in data_files if _label_in_name(Path(f.lower()).stem, subset_lower)]
+ named = [f for f in data_files if _label_in_name(_name_parts(f.lower())[1], subset_lower)]
if any(_split_rank(f, split_lower) <= 1 for f in named):
return named
if any(_split_rank(f, split_lower) <= 1 for f in data_files):
@@ -527,7 +550,7 @@ def _builder_exts(suffix: str) -> tuple[str, ...]:
def _of_suffix(paths: list[str], suffix: str) -> list[str]:
group = _builder_exts(suffix)
- return [p for p in paths if Path(p).suffix.lower() in group]
+ return [p for p in paths if _name_parts(p)[0].lower() in group]
def _extension_glob(paths: list[str], suffix: str, repo_files: list[str]) -> str:
diff --git a/studio/backend/tests/test_seed_path_split_fast.py b/studio/backend/tests/test_seed_path_split_fast.py
new file mode 100644
index 00000000000..99ce17e4b16
--- /dev/null
+++ b/studio/backend/tests/test_seed_path_split_fast.py
@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The string-split seed path helpers equal PurePosixPath / Path / a fresh re.search, POSIX and Windows flavours."""
+
+import importlib.util
+import random
+import re
+from pathlib import Path, PurePosixPath, PureWindowsPath
+
+import pytest
+
+
+def _load(monkeypatch):
+ pytest.importorskip("fastapi")
+ pytest.importorskip("structlog")
+ backend_root = Path(__file__).resolve().parent.parent
+ monkeypatch.syspath_prepend(str(backend_root))
+ spec = importlib.util.spec_from_file_location(
+ "seed_path_split_under_test", backend_root / "routes" / "data_recipe" / "seed.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+_SEGMENTS = [
+ "data",
+ "train",
+ "Train",
+ "TEST",
+ "validation",
+ "dev",
+ "__hidden",
+ "__init__",
+ ".git",
+ ".",
+ "..",
+ "",
+ "sub-train",
+ "en",
+ "sample-10BT",
+ "x y",
+ "a:b",
+ "C:",
+ "back\\slash",
+ "İstanbul",
+ "é",
+ "-",
+ "00",
+]
+_NAMES = [
+ "train",
+ "test",
+ "train-00000-of-00004",
+ "questions.train",
+ "a.csv",
+ ".hidden",
+ "README",
+ "x.",
+ "y..",
+ "..",
+ ".",
+ "TRAIN",
+ "en-train",
+ "000_00001",
+ "__train",
+ "tést",
+ "dev-1",
+ "a b",
+ "c:d",
+]
+_EXTS = [
+ ".parquet",
+ ".PARQUET",
+ ".json",
+ ".jsonl",
+ ".csv",
+ ".json.gz",
+ "",
+ ".",
+ ".parquet/",
+ ".csv.parquet",
+]
+
+
+def _paths(count = 3000, seed = 0):
+ rng = random.Random(seed)
+ out = [
+ "",
+ ".",
+ "..",
+ "/",
+ "//",
+ "///",
+ "a",
+ "a/",
+ "a/.",
+ "a/..",
+ "./a",
+ "/a",
+ "//a/b",
+ "a//b",
+ "a/./b",
+ "C:/a.parquet",
+ "C:a.parquet",
+ "a\\b.parquet",
+ "a/b.parquet ",
+ "a/b .parquet",
+ "a/.parquet",
+ "a/b.",
+ "a/b..parquet",
+ "data/train-00000-of-00001.parquet",
+ "İ.PARQUET",
+ "data/x.Parquet",
+ ]
+ for _ in range(count):
+ parts = [rng.choice(_SEGMENTS) for _ in range(rng.choice([0, 1, 1, 2, 3]))]
+ path = "/".join(parts + [rng.choice(_NAMES) + rng.choice(_EXTS)])
+ if rng.random() < 0.05:
+ path = "/" + path
+ if rng.random() < 0.05:
+ path = path.replace("/", "//", 1)
+ if rng.random() < 0.03:
+ path += "/"
+ out.append(path)
+ return out
+
+
+def test_marked_parts_and_loader_facts_match_pure_posix_path(monkeypatch):
+ seed = _load(monkeypatch)
+ for path in _paths():
+ parsed = PurePosixPath(path)
+ for prefix in (".", "__"):
+ for which, parts in (("parts", parsed.parts), ("parent_parts", parsed.parent.parts)):
+ expected = [p for p in parts if p.startswith(prefix) and set(p) != {"."}]
+ assert seed._marked_parts(path, prefix, which) == expected, (path, prefix, which)
+ assert seed._loader_path_facts(path)[2] == parsed.name, path
+
+
[email protected]("flavour", [PurePosixPath, PureWindowsPath])
+def test_name_parts_match_pathlib(monkeypatch, flavour):
+ seed = _load(monkeypatch)
+ monkeypatch.setattr(seed, "Path", flavour)
+ for path in _paths():
+ for value in (path, path.lower()):
+ parsed = flavour(value)
+ assert seed._name_parts(value) == (parsed.suffix, parsed.stem), (flavour, value)
+
+
+def test_label_matcher_matches_a_fresh_search(monkeypatch):
+ seed = _load(monkeypatch)
+ labels = ["train", "dev", "en", "sample-10bt", "a.b", "x+y", "[", "", "validation"]
+ for path in _paths(500):
+ name = path.lower()
+ for label in labels:
+ expected = (
+ re.search(rf"(?:^|{seed._SEP}){re.escape(label)}(?:$|{seed._SEP})", name)
+ is not None
+ )
+ assert seed._label_in_name(name, label) == expected, (name, label)
+
+
[email protected]("flavour", [PurePosixPath, PureWindowsPath])
+def test_split_rank_and_suffix_filter_match_pathlib(monkeypatch, flavour):
+ seed = _load(monkeypatch)
+ monkeypatch.setattr(seed, "Path", flavour)
+ paths = _paths(1500, seed = 1)
+ for path in paths:
+ lowered = path.lower()
+ labels = seed._split_labels("train")
+ if seed._split_folder(lowered, labels):
+ expected = 0
+ elif any(seed._label_in_name(flavour(lowered).stem, label) for label in labels):
+ expected = 1
+ else:
+ expected = 2
+ assert seed._split_rank(path, "train") == expected, (flavour, path)
+ for suffix in (".parquet", ".json", ".JSONL", ".csv", ""):
+ group = seed._builder_exts(suffix)
+ expected = [p for p in paths if flavour(p).suffix.lower() in group]
+ assert seed._of_suffix(paths, suffix) == expected, (flavour, suffix)