Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: split seed Hub file names by string and compile split label matchers once (#16352)
Daniel Hane811e74b3d3e
diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.pyindex bc035f1ef28..f3cd3d6fe72 100644--- a/studio/backend/routes/data_recipe/seed.py+++ b/studio/backend/routes/data_recipe/seed.py@@ -8,6 +8,7 @@ from __future__ import annotationsfrom core.training.account_jobs import account_path, managed_accountimport base64import binascii+import functoolsimport jsonimport osimport re@@ -302,9 +303,26 @@ def _class_end(pattern: str, start: int) -> int:return closing if closing > start else -1+def _posix_tail(path: str) -> list[str]:+ """PurePosixPath(path).parts without the root: split on "/", empty and "." parts dropped."""+ return [part for part in path.split("/") if part and part != "."]+++def _name_parts(path: str) -> tuple[str, str]:+ """Path(path).suffix and .stem; by hand only for plain relative ASCII names, which every pathlib flavour and version reads alike."""+ if path.isascii() and not path.startswith("/") and "\\" not in path and ":" not in path:+ name = path[path.rfind("/") + 1 :]+ dot = name.rfind(".")+ if name[:1] not in ("", ".") and dot < len(name) - 1:+ return (name[dot:], name[:dot]) if dot > 0 else ("", name)+ parsed = Path(path)+ return parsed.suffix, parsed.stem++def _marked_parts(path: str, prefix: str, which: str) -> list[str]:- parsed = PurePosixPath(path)- parts = parsed.parts if which == "parts" else parsed.parent.parts+ parts = _posix_tail(path)+ if which != "parts":+ parts = parts[:-1]return [part for part in parts if part.startswith(prefix) and set(part) != {"."}]@@ -338,7 +356,7 @@ def _loader_path_facts(path: str) -> tuple[int, int, str]:return (len(_marked_parts(path, ".", "parts")),len(_marked_parts(path, "__", "parent_parts")),- PurePosixPath(path).name,+ (_posix_tail(path) or [""])[-1],)@@ -366,7 +384,12 @@ def _common_parent(paths: list[str]) -> str:def _label_in_name(name: str, label: str) -> bool:"""Match the label in place, not by tokens, so sample-10BT survives; digits count as separators: train1 is train, training is not."""- return re.search(rf"(?:^|{_SEP}){re.escape(label)}(?:$|{_SEP})", name) is not None+ return _label_matcher(label).search(name) is not None++[email protected]_cache(maxsize = 256)+def _label_matcher(label: str) -> re.Pattern[str]:+ return re.compile(rf"(?:^|{_SEP}){re.escape(label)}(?:$|{_SEP})")def _split_folder(lowered_path: str, labels: tuple[str, ...]) -> bool:@@ -401,7 +424,7 @@ def _split_rank(path: str, split_lower: str) -> int:if _split_folder(lowered, labels):return 0# Only the final extension comes off: questions.train.parquet keeps its split.- stem = Path(lowered).stem+ stem = _name_parts(lowered)[1]if any(_label_in_name(stem, label) for label in labels):return 1return 2@@ -418,7 +441,7 @@ def _in_subset(data_files: list[str], subset: str | None, split_lower: str) -> l]if in_folder:return in_folder- named = [f for f in data_files if _label_in_name(Path(f.lower()).stem, subset_lower)]+ named = [f for f in data_files if _label_in_name(_name_parts(f.lower())[1], subset_lower)]if any(_split_rank(f, split_lower) <= 1 for f in named):return namedif any(_split_rank(f, split_lower) <= 1 for f in data_files):@@ -527,7 +550,7 @@ def _builder_exts(suffix: str) -> tuple[str, ...]:def _of_suffix(paths: list[str], suffix: str) -> list[str]:group = _builder_exts(suffix)- return [p for p in paths if Path(p).suffix.lower() in group]+ return [p for p in paths if _name_parts(p)[0].lower() in group]def _extension_glob(paths: list[str], suffix: str, repo_files: list[str]) -> str:diff --git a/studio/backend/tests/test_seed_path_split_fast.py b/studio/backend/tests/test_seed_path_split_fast.pynew file mode 100644index 00000000000..99ce17e4b16--- /dev/null+++ b/studio/backend/tests/test_seed_path_split_fast.py@@ -0,0 +1,182 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The string-split seed path helpers equal PurePosixPath / Path / a fresh re.search, POSIX and Windows flavours."""++import importlib.util+import random+import re+from pathlib import Path, PurePosixPath, PureWindowsPath++import pytest+++def _load(monkeypatch):+ pytest.importorskip("fastapi")+ pytest.importorskip("structlog")+ backend_root = Path(__file__).resolve().parent.parent+ monkeypatch.syspath_prepend(str(backend_root))+ spec = importlib.util.spec_from_file_location(+ "seed_path_split_under_test", backend_root / "routes" / "data_recipe" / "seed.py"+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++_SEGMENTS = [+ "data",+ "train",+ "Train",+ "TEST",+ "validation",+ "dev",+ "__hidden",+ "__init__",+ ".git",+ ".",+ "..",+ "",+ "sub-train",+ "en",+ "sample-10BT",+ "x y",+ "a:b",+ "C:",+ "back\\slash",+ "İstanbul",+ "é",+ "-",+ "00",+]+_NAMES = [+ "train",+ "test",+ "train-00000-of-00004",+ "questions.train",+ "a.csv",+ ".hidden",+ "README",+ "x.",+ "y..",+ "..",+ ".",+ "TRAIN",+ "en-train",+ "000_00001",+ "__train",+ "tést",+ "dev-1",+ "a b",+ "c:d",+]+_EXTS = [+ ".parquet",+ ".PARQUET",+ ".json",+ ".jsonl",+ ".csv",+ ".json.gz",+ "",+ ".",+ ".parquet/",+ ".csv.parquet",+]+++def _paths(count = 3000, seed = 0):+ rng = random.Random(seed)+ out = [+ "",+ ".",+ "..",+ "/",+ "//",+ "///",+ "a",+ "a/",+ "a/.",+ "a/..",+ "./a",+ "/a",+ "//a/b",+ "a//b",+ "a/./b",+ "C:/a.parquet",+ "C:a.parquet",+ "a\\b.parquet",+ "a/b.parquet ",+ "a/b .parquet",+ "a/.parquet",+ "a/b.",+ "a/b..parquet",+ "data/train-00000-of-00001.parquet",+ "İ.PARQUET",+ "data/x.Parquet",+ ]+ for _ in range(count):+ parts = [rng.choice(_SEGMENTS) for _ in range(rng.choice([0, 1, 1, 2, 3]))]+ path = "/".join(parts + [rng.choice(_NAMES) + rng.choice(_EXTS)])+ if rng.random() < 0.05:+ path = "/" + path+ if rng.random() < 0.05:+ path = path.replace("/", "//", 1)+ if rng.random() < 0.03:+ path += "/"+ out.append(path)+ return out+++def test_marked_parts_and_loader_facts_match_pure_posix_path(monkeypatch):+ seed = _load(monkeypatch)+ for path in _paths():+ parsed = PurePosixPath(path)+ for prefix in (".", "__"):+ for which, parts in (("parts", parsed.parts), ("parent_parts", parsed.parent.parts)):+ expected = [p for p in parts if p.startswith(prefix) and set(p) != {"."}]+ assert seed._marked_parts(path, prefix, which) == expected, (path, prefix, which)+ assert seed._loader_path_facts(path)[2] == parsed.name, path++[email protected]("flavour", [PurePosixPath, PureWindowsPath])+def test_name_parts_match_pathlib(monkeypatch, flavour):+ seed = _load(monkeypatch)+ monkeypatch.setattr(seed, "Path", flavour)+ for path in _paths():+ for value in (path, path.lower()):+ parsed = flavour(value)+ assert seed._name_parts(value) == (parsed.suffix, parsed.stem), (flavour, value)+++def test_label_matcher_matches_a_fresh_search(monkeypatch):+ seed = _load(monkeypatch)+ labels = ["train", "dev", "en", "sample-10bt", "a.b", "x+y", "[", "", "validation"]+ for path in _paths(500):+ name = path.lower()+ for label in labels:+ expected = (+ re.search(rf"(?:^|{seed._SEP}){re.escape(label)}(?:$|{seed._SEP})", name)+ is not None+ )+ assert seed._label_in_name(name, label) == expected, (name, label)++[email protected]("flavour", [PurePosixPath, PureWindowsPath])+def test_split_rank_and_suffix_filter_match_pathlib(monkeypatch, flavour):+ seed = _load(monkeypatch)+ monkeypatch.setattr(seed, "Path", flavour)+ paths = _paths(1500, seed = 1)+ for path in paths:+ lowered = path.lower()+ labels = seed._split_labels("train")+ if seed._split_folder(lowered, labels):+ expected = 0+ elif any(seed._label_in_name(flavour(lowered).stem, label) for label in labels):+ expected = 1+ else:+ expected = 2+ assert seed._split_rank(path, "train") == expected, (flavour, path)+ for suffix in (".parquet", ".json", ".JSONL", ".csv", ""):+ group = seed._builder_exts(suffix)+ expected = [p for p in paths if flavour(p).suffix.lower() in group]+ assert seed._of_suffix(paths, suffix) == expected, (flavour, suffix)