Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: use only the chosen split and subset for Hugging Face recipe data (#11294)
Nilayca89af7a7211
diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.pyindex 4b9a58c64c7..37c23dae1d7 100644--- a/studio/backend/routes/data_recipe/seed.py+++ b/studio/backend/routes/data_recipe/seed.py@@ -13,7 +13,7 @@ import osimport reimport shutilfrom itertools import islice-from pathlib import Path+from pathlib import Path, PurePosixPathfrom typing import Anyfrom uuid import uuid4@@ -22,6 +22,14 @@ from fastapi import APIRouter, Depends, HTTPException, UploadFile, File as FastAfrom auth.authentication import allow_ambient_hf_tokenfrom core.data_recipe.jsonable import to_preview_jsonable+from hub.services.datasets.local_options import (+ _sharded_splits,+ _IGNORED_DATA_FILENAMES,+ _MAX_MODULE_INFERENCE_FILES,+ _METADATA_FILENAMES,+ _SEP,+ _SPLIT_KEYWORDS,+)from hub.utils.dataset_cache import refuse_unauthorized_dataset_previewfrom hub.utils.hf_tokens import HfTokenArg, hf_token_argfrom loggers import get_logger@@ -68,6 +76,7 @@ def _chunking() -> Any:DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv")DEFAULT_SPLIT = "train"+DEFAULT_CONFIG = "default"LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"}UNSTRUCTURED_ALLOWED_EXTS = {".pdf", ".docx", ".txt", ".md"}SEED_UPLOAD_DIR = LazyPath(seed_uploads_root)@@ -101,7 +110,10 @@ def _normalize_optional_text(value: str | None) -> str | None:return trimmed if trimmed else None-def _list_hf_data_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:+def _list_hf_repo_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:+ """Every file in the repo: the reader globs over all of them, not just the+ ones a builder can load, so a pattern is judged against the whole listing.+ """try:from huggingface_hub import HfApifrom huggingface_hub.utils import HfHubHTTPError@@ -109,51 +121,738 @@ def _list_hf_data_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:return []try:api = HfApi(token = token)- repo_files = api.list_repo_files(dataset_name, repo_type = "dataset", token = token)- return [file for file in repo_files if file.lower().endswith(DATA_EXTS)]+ return api.list_repo_files(dataset_name, repo_type = "dataset", token = token)except (HfHubHTTPError, OSError, ValueError):return []-def _select_best_file(data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:+def _list_hf_data_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:+ repo_files = _list_hf_repo_files(dataset_name = dataset_name, token = token)+ return [file for file in repo_files if file.lower().endswith(DATA_EXTS)]+++def _list_hf_dataset_configs(*, dataset_name: str, token: HfTokenArg) -> list[dict[str, Any]]:+ """The `configs:` block of the dataset card, which maps a config to its own file globs.++ A config name is not required to be a folder name: fineweb-edu's `sample-10BT`+ lives under `sample/10BT/`, so a folder-name guess reads a different config.+ """+ try:+ from huggingface_hub import HfApi+ from huggingface_hub.utils import HfHubHTTPError+ except ImportError:+ return []+ try:+ api = HfApi(token = token)+ card_data = api.dataset_info(dataset_name, token = token).card_data+ configs = (card_data or {}).get("configs") if card_data is not None else None+ except (HfHubHTTPError, OSError, ValueError, AttributeError):+ return []+ return [config for config in (configs or []) if isinstance(config, dict)]+++def _as_pattern_list(paths: Any) -> list[str]:+ if isinstance(paths, str):+ return [paths]+ if isinstance(paths, list):+ return [path for path in paths if isinstance(path, str)]+ return []+++def _patterns_for_split(+ data_files: Any,+ split: str,+ *,+ exact: bool = True,+) -> list[str]:+ """Every shape a card may use for `data_files`, as hub/services/datasets does.++ A bare string or list of strings is the shorthand for the train split. The+ name is read exactly first: a card may declare Train beside train, and+ `load_dataset` is handed the name the caller asked for, not a folded one.+ """+ if isinstance(data_files, str):+ return [data_files] if _same_split(DEFAULT_SPLIT, split, exact) else []+ if isinstance(data_files, dict):+ for name, paths in data_files.items():+ if _same_split(str(name), split, exact):+ return _as_pattern_list(paths)+ return []+ if isinstance(data_files, list):+ # Entry by entry: a list may declare one split more than once, and+ # stopping at the first match drops the rest of it.+ patterns: list[str] = []+ for entry in data_files:+ if isinstance(entry, str):+ if _same_split(DEFAULT_SPLIT, split, exact):+ patterns.append(entry)+ elif isinstance(entry, dict) and _same_split(+ str(entry.get("split") or ""), split, exact+ ):+ patterns.extend(_as_pattern_list(entry.get("path")))+ return patterns+ return []+++def _same_split(declared: str, split: str, exact: bool) -> bool:+ return declared == split if exact else declared.lower() == split.lower()+++def _declared_split_patterns(+ configs: list[dict[str, Any]],+ split: str = DEFAULT_SPLIT,+ subset: str | None = None,+) -> list[str]:+ """The globs the card declares for this split, under the named config."""+ config = _pick_config(configs, subset)+ if config is None:+ return []+ declared = config.get("data_files")+ named = _patterns_for_split(declared, split) or _patterns_for_split(+ declared, split, exact = False+ )+ folder = _config_folder(config)+ # `_resolve_data_files` joins `data_dir` to the declared pattern whatever+ # the two say, even where the pattern repeats the folder (`builder.py`).+ return [_normalized_glob(f"{folder}/{p}" if folder else p) for p in named]+++def _normalized_glob(pattern: str) -> str:+ """The pattern as the listing spells it, the way `normpath` would.++ `list_repo_files` returns data/train.parquet, so ./data and staging/../data+ have to lose their dots here or match nothing.+ """+ parts: list[str] = []+ for part in pattern.strip().split("/"):+ if part in ("", "."):+ continue+ if part == ".." and parts and parts[-1] != "..":+ parts.pop()+ continue+ parts.append(part)+ return "/".join(parts)+++def _config_folder(config: dict[str, Any]) -> str:+ folder = config.get("data_dir")+ return _normalized_glob(folder) if isinstance(folder, str) else ""+++def _config_scope(configs: list[dict[str, Any]], subset: str | None) -> str:+ """The folder a config is confined to, for a card that names one and lists no files."""+ config = _pick_config(configs, subset)+ return _config_folder(config) if config else ""+++def _pick_config(configs: list[dict[str, Any]], subset: str | None) -> dict[str, Any] | None:+ """The config a bare request lands on, the way the Hub resolves it.++ Without a subset the answer is not always the one literally called `default`:+ a card may flag another config as the default, and a card with a single+ config uses it whatever its name (imdb ships only `plain_text`).+ """+ if subset:+ # A card may carry both Foo and foo, so the exact one wins and a folded+ # match is only a fallback. No config_name means the default one.+ names = [(c, str(c.get("config_name") or DEFAULT_CONFIG)) for c in configs]+ return next(+ (c for c, name in names if name == subset),+ next((c for c, name in names if name.lower() == subset.lower()), None),+ )+ flagged = next((c for c in configs if c.get("default") is True), None)+ if flagged is not None:+ return flagged+ if len(configs) == 1:+ return configs[0]+ # Exactly `default`: Default and DEFAULT are configs to be asked for.+ return next(+ (c for c in configs if str(c.get("config_name") or DEFAULT_CONFIG) == DEFAULT_CONFIG),+ None,+ )+++def _glob_to_regex(pattern: str) -> re.Pattern[str]:+ """A card glob as a regex: `*` stops at a folder boundary, `**` crosses it.++ Matching on the literal prefix instead would let `data/train-*.parquet` claim+ `data/train-notes.json`, which the recipe path then never reads.+ """+ parts: list[str] = []+ i = 0+ while i < len(pattern):+ char = pattern[i]+ if char == "*" and pattern[i + 1 : i + 2] == "*":+ if pattern[i + 2 : i + 3] == "/":+ # Whole folders, or none: `data/**/train.*` is not `data/nottrain.x`.+ parts.append("(?:[^/]+/)*")+ i += 3+ else:+ parts.append(".*")+ i += 2+ continue+ if char == "*":+ parts.append("[^/]*")+ elif char == "?":+ parts.append("[^/]")+ elif char == "[":+ end = _class_end(pattern, i)+ if end < 0:+ parts.append(r"\[")+ else:+ body = pattern[i + 1 : end].replace("\\", "\\\\")+ # Only `!` negates a glob class; a leading `^` is a character.+ if body[:1] == "!":+ body = f"^{body[1:]}"+ elif body[:1] == "^":+ body = f"\\^{body[1:]}"+ parts.append(f"[{body}]")+ i = end+ else:+ parts.append(re.escape(char))+ i += 1+ return re.compile("".join(parts) + r"\Z")+++def _class_end(pattern: str, start: int) -> int:+ """Index of the `]` closing the class opened at `start`, or -1 if unclosed."""+ i = start + 1+ if pattern[i : i + 1] in ("!", "^"):+ i += 1+ if pattern[i : i + 1] == "]":+ i += 1+ closing = pattern.find("]", i)+ return closing if closing > start else -1+++def _ignored_by_the_loader(path: str, pattern: str) -> bool:+ """Whether the loader drops this match by name, as dataset_info.json.++ Those basenames leave a glob's matches unless the pattern is that file+ itself (`data_files.FILES_TO_IGNORE`).+ """+ name = PurePosixPath(path).name+ return name in _IGNORED_DATA_FILENAMES and PurePosixPath(pattern).name != name+++def _hidden_to_the_loader(path: str, pattern: str) -> bool:+ """Whether a match sits in a part the loader skips unless it was asked for.++ A hidden or dunder component the pattern does not carry itself drops the+ match, so `**/*` never reaches .backup/ or __pycache__/ (`data_files`).+ """+ # A dot counts anywhere, a dunder only in a parent, so data/__train.json+ # is a file the loader reads.+ for prefix, parts in ((".", "parts"), ("__", "parent_parts")):+ in_path = _marked_parts(path, prefix, parts)+ in_pattern = _marked_parts(pattern, prefix, parts)+ if len(in_path) != len(in_pattern):+ return True+ return False+++def _marked_parts(path: str, prefix: str, which: str) -> list[str]:+ parsed = PurePosixPath(path)+ parts = parsed.parts if which == "parts" else parsed.parent.parts+ return [part for part in parts if part.startswith(prefix) and set(part) != {"."}]+++def _files_under_patterns(+ patterns: list[str],+ data_files: list[str],+ *,+ as_the_loader_would: bool = False,+) -> list[str]:+ """The files a glob takes, under the loader's rules for a card's own glob.++ A pattern this module writes goes to a reader that skips nothing, so it is+ judged on what it plainly matches.+ """+ matchers = [(pattern, _glob_to_regex(pattern)) for pattern in patterns]+ return [+ f+ for f in data_files+ if any(+ m.match(f)+ and not (+ as_the_loader_would+ and (_hidden_to_the_loader(f, p) or _ignored_by_the_loader(f, p))+ )+ for p, m in matchers+ )+ ]+++def _common_parent(paths: list[str]) -> str:+ parts = [Path(path).parent.as_posix().split("/") for path in paths]+ shared: list[str] = []+ for segments in zip(*parts):+ if len(set(segments)) != 1 or segments[0] == ".":+ break+ shared.append(segments[0])+ return "/".join(shared)+++def _label_in_name(name: str, label: str) -> bool:+ """The label standing on its own in a file name, separators and all.++ Tokenizing the name instead would lose a label that carries a separator, so+ validation_matched or sample-10BT could never match anything. The separators+ are the loader's own (`local_options._SEP`), digits included, so train1 is+ the train split while training is not.+ """+ return re.search(rf"(?:^|{_SEP}){re.escape(label)}(?:$|{_SEP})", name) is not None+++def _split_folder(lowered_path: str, labels: tuple[str, ...]) -> bool:+ """A folder standing for this split, qualified or not: train, train_a, en-train.++ The loader reads a separator-qualified folder as the split too+ (`local_options._DIR_NAME_KEYWORD_PATTERNS`), so an exact component match+ would leave train_a ranking as nothing.+ """+ return any(+ _label_in_name(part, label) for part in lowered_path.split("/")[:-1] for label in labels+ )+++def _split_labels(split_lower: str) -> tuple[str, ...]:+ """The names `datasets` accepts for this split: dev is validation, training is train."""+ for canonical, aliases in _SPLIT_KEYWORDS.items():+ if split_lower == canonical or split_lower in aliases:+ return aliases+ return (split_lower,)+++def _carries_another_split(path: str, split_lower: str) -> bool:+ """The path is labelled with a split, and it is not the one being asked for."""+ wanted = set(_split_labels(split_lower))+ return any(+ canonical not in wanted and _split_rank(path, canonical) <= 1+ for canonical in _SPLIT_KEYWORDS+ )+++def _split_rank(path: str, split_lower: str) -> int:+ """0 the split is a folder, 1 the file name carries it, 2 neither.++ The name is read as whole labels rather than by where the split sits in it,+ so questions_train_000.jsonl counts as train, and `datasets` own aliases+ count too, so dev.jsonl is validation.+ """+ lowered = path.lower()+ labels = _split_labels(split_lower)+ if _split_folder(lowered, labels):+ return 0+ # Only the final extension comes off: questions.train.parquet keeps its split.+ stem = Path(lowered).stem+ if any(_label_in_name(stem, label) for label in labels):+ return 1+ return 2+++def _in_subset(data_files: list[str], subset: str | None, split_lower: str) -> list[str]:+ """The files carrying this config label, as a folder or in the file name.++ A repo may encode its configs in the names instead of the layout, as+ main-train.parquet beside socratic-train.parquet. A folder carrying the+ label is the config whatever its files are called; a name only carrying it+ while belonging to no part of the split is a coincidence, not the config,+ so it is dropped rather than allowed to decide the answer.+ """+ if not subset:+ return data_files+ subset_lower = subset.lower()+ # Exactly first: a repo may hold Foo/ beside foo/, and folding the two+ # together would read both configs as one.+ in_folder = [f for f in data_files if subset in f.split("/")[:-1]] or [+ f for f in data_files if subset_lower in f.lower().split("/")[:-1]+ ]+ if in_folder:+ return in_folder+ named = [f for f in data_files if _label_in_name(Path(f.lower()).stem, subset_lower)]+ if any(_split_rank(f, split_lower) <= 1 for f in named):+ return named+ if any(_split_rank(f, split_lower) <= 1 for f in data_files):+ return data_files+ return named or data_files+++def _select_best_file(+ data_files: list[str],+ split: str = DEFAULT_SPLIT,+ subset: str | None = None,+) -> str | None:if not data_files:return Nonesplit_lower = split.lower()+ return sorted(+ _in_subset(data_files, subset, split_lower),+ key = lambda p: (_split_rank(p, split_lower), len(p)),+ )[0]+++def _pattern_fits_the_split(+ data_files: list[str], wanted: set[str], root: str, pattern: str+) -> bool:+ """The pattern must take every wanted file under the root, and no other file there.++ A folder may name one split several ways at once (train-part.parquet beside+ questions_train.parquet), so a glob built from whichever file was picked can+ drop the rest; a loose glob can just as easily swallow a neighbouring split.+ Read relative to the root the pattern is anchored at, so a split spread over+ sibling folders is only accepted by a pattern that reaches all of them.+ Judged against the WHOLE listing, since a candidate checked only against the+ subset slice can still match another subset's files.+ """+ matcher = _glob_to_regex(pattern)+ prefix = f"{root}/" if root and root != "." else ""+ for path in data_files:+ if not path.startswith(prefix):+ continue+ if bool(matcher.match(path[len(prefix) :])) != (path in wanted):+ return False+ return True+- def score(path: str) -> tuple[int, int]:- name = path.lower()- if f"/{split_lower}/" in name:- return (0, len(path))- if (- f"_{split_lower}." in name- or f"-{split_lower}." in name- or f"/{split_lower}." in name- or f"/{split_lower}_" in name- or f"/{split_lower}-" in name- ):- return (1, len(path))- return (2, len(path))+def _candidate_patterns(stem: str, suffix: str, split_lower: str) -> list[str]:+ """Globs for this split's files in one folder, narrowest first.++ Built from whichever of the split's names this file actually uses, since a+ validation shard may well be called dev.+ """+ stem_lower = stem.lower()+ candidates: list[str] = []+ for label in _split_labels(split_lower):+ if stem_lower.startswith((f"{label}-", f"{label}_", f"{label}.")):+ candidates.append(f"{stem[: len(label) + 1]}*{suffix}")+ if stem_lower == label or stem_lower.endswith((f"_{label}", f"-{label}")):+ # Keep the star: train.jsonl may be sharded beside train_2.jsonl.+ candidates.append(f"{stem}*{suffix}")+ at = stem_lower.find(label)+ if at >= 0:+ candidates.append(f"*{stem[at : at + len(label)]}*{suffix}")+ # Last and narrowest: the whole name, which keeps train-main off+ # train-socratic.+ candidates.append(f"{stem}*{suffix}")+ return list(dict.fromkeys(candidates))+++def _with_data_extension(pattern: str, suffix: str) -> str:+ """A card glob ends where the file name does; the reader picks by extension."""+ current = Path(pattern).suffix+ if current.lower() in DATA_EXTS:+ return pattern+ if current and any(char in current for char in "*?["):+ pattern = pattern[: -len(current)]+ # fsspec refuses data/**.parquet: `**` has to be a whole component.+ if pattern.endswith("**"):+ pattern = f"{pattern}/*"+ return f"{pattern}{suffix}"+++def _ext_rank(suffix: str) -> int:+ """Where `datasets` puts this extension in a draw: parquet, jsonl, json, csv."""+ return -DATA_EXTS.index(suffix) if suffix in DATA_EXTS else -99+++def _dominant_suffix(paths: list[str]) -> str:+ """The format the loader would build this split from: most files, parquet on ties.++ One broad card glob can cover several formats, and picking whichever name+ sorts first would point the recipe at a format the split does not use+ (`local_options._one_module`).+ """+ counts: dict[str, int] = {}+ # The window first, exactly as the loader slices it, and metadata dropped+ # only within it: a shard past the window is one the loader never sees.+ window = sorted(paths)[:_MAX_MODULE_INFERENCE_FILES]+ voting = [p for p in window if Path(p).name.lower() not in _METADATA_FILENAMES] or window+ for path in voting:+ # A vote for every suffix in the name and one extension per vote, as+ # `load.infer_module_for_data_files_list` counts them: a.csv.parquet is+ # a csv as much as a parquet, and .json lends .jsonl none of its votes.+ for part in Path(path).name.lower().split(".")[1:]:+ counts[f".{part}"] = counts.get(f".{part}", 0) + 1+ if not counts:+ return ""++ # The builder widens only once its extension has won on its own. A winner no+ # file actually ends in cannot be written as a glob, so the next one that can+ # is taken rather than a pattern that matches nothing.+ for winner in sorted(counts, key = lambda s: (counts[s], _ext_rank(s)), reverse = True):+ group = _builder_exts(winner)+ named = next((Path(p).suffix for p in paths if Path(p).suffix.lower() in group), "")+ if named:+ return named+ return ""+++# Extensions one builder reads: `datasets` loads .json and .jsonl with the same one.+_BUILDER_EXTS = ((".json", ".jsonl"),)+++def _builder_exts(suffix: str) -> tuple[str, ...]:+ lowered = suffix.lower()+ for group in _BUILDER_EXTS:+ if lowered in group:+ return group+ return (lowered,)+++def _of_suffix(paths: list[str], suffix: str) -> list[str]:+ group = _builder_exts(suffix)+ return [p for p in paths if Path(p).suffix.lower() in group]+++def _extension_glob(paths: list[str], suffix: str, repo_files: list[str]) -> str:+ """What the pattern should end in to keep every file this builder reads.++ A split over both .json and .jsonl needs the shared start of the two, which+ leaves the glob open at the end; where the repo holds a a.json.gz that glob+ would read it too, so there the winning extension is used alone.+ """+ group = set(_builder_exts(suffix))+ used = {Path(path).suffix.lower() for path in paths} & group+ if len(used) < 2:+ return suffix+ stem = os.path.commonprefix(sorted(used))+ matcher = re.compile(rf"{re.escape(stem)}[^/]*\Z", re.IGNORECASE)+ # Only what the emitted pattern can reach: it is anchored at the folder the+ # declared files share, so a name collision anywhere else is not its problem.+ scope = _common_parent(paths)+ prefix = f"{scope}/" if scope else ""+ if any(+ path.startswith(prefix)+ and Path(path).suffix.lower() not in group+ and matcher.search(Path(path).name)+ for path in repo_files+ ):+ return suffix+ return f"{stem}*"+++def _common_name_prefix(paths: list[str]) -> str:+ """What the file names share before the extension, which the glob supplies."""+ return os.path.commonprefix([Path(path).stem for path in paths]) if paths else ""+++def _name_initials(paths: list[str]) -> str:+ """The first characters of these file names, as a glob class body.++ Only letters and digits, so nothing in the class can be read as syntax.+ """+ initials = sorted({Path(path).name[:1] for path in paths if Path(path).name[:1].isalnum()})+ if not initials or len(initials) != len({Path(path).name[:1] for path in paths}):+ return ""+ return "".join(initials)+++def _staged_split_files(+ data_files: list[str],+ labels: tuple[str, ...],+ scope: str = "",+) -> set[str]:+ """This split's shards, when the repo is laid out the way the loader looks first.++ Split inference is staged and the sharded names come first: once+ data/train-00000-of-00001.parquet is there, a train-named file elsewhere is+ not in the split (`local_options._grouped_splits`).+ """+ # The loader reads a config's files relative to its `data_dir`, and the+ # sharded names it looks for start at `data/`, so the scope comes off first.+ prefix = f"{scope}/" if scope else ""+ inside = [path[len(prefix) :] for path in data_files if path.startswith(prefix)]+ grouped = _sharded_splits([PurePosixPath(path) for path in inside])+ if not grouped:+ return set()+ staged = {+ f"{prefix}{path.as_posix()}"+ for split, paths in grouped.items()+ if split.lower() in labels+ for path in paths+ }+ return staged if staged < set(data_files) else set()- return sorted(data_files, key = score)[0]++def _anchor(dataset_name: str, root: str) -> str:+ """The recipe path up to the folder a pattern is written relative to."""+ base = f"datasets/{dataset_name}"+ return f"{base}/{root}" if root and root != "." else base+++def _name_finals(paths: list[str]) -> str:+ """The last characters of these names before the extension, as a class body."""+ finals = {Path(path).stem[-1:] for path in paths}+ return "".join(sorted(finals)) if finals and all(c.isalnum() for c in finals) else ""+++def _widened_declared_pattern(+ declared_files: list[str],+ data_files: list[str],+ split_lower: str,+ suffix: str,+ *,+ fallback_glob: bool = False,+) -> str:+ """One glob covering several declared ones, keeping the split out of it if it can.++ The folder holding them all may hold the other splits too, which is what the+ split-named form avoids; it is only used when it takes the declared files+ and nothing else.+ """+ parent = _common_parent(declared_files)+ base = f"{parent}/**" if parent else "**"+ prefix = f"{parent}/" if parent else ""+ wanted = set(declared_files)+ # The split may be written into the file names or into the folders between+ # here and them, so try both before giving up on keeping it.+ # Whichever of the split's names the files use: a validation set may well be+ # declared as dev-*.parquet.+ candidates = [f"{base}/*{label}*{suffix}" for label in _split_labels(split_lower)]+ candidates += [f"{prefix}**/*{label}*/**/*{suffix}" for label in _split_labels(split_lower)]+ # Declared files that say nothing about the split may still share a name the+ # neighbouring splits do not, as data/part-a beside data/part-b, or+ # sets/a/part beside sets/b/part with a test file in sets/c.+ shared_name = _common_name_prefix(declared_files)+ if shared_name:+ candidates.append(f"{prefix}{shared_name}*{suffix}")+ candidates.append(f"{prefix}**/{shared_name}*{suffix}")+ # Unrelated names still start somewhere the neighbours do not. A class is the+ # only union the reader understands; it rejects `{a,b}` outright.+ initials = _name_initials(declared_files)+ finals = _name_finals(declared_files)+ if initials:+ candidates.append(f"{prefix}**/[{initials}]*{suffix}")+ # Names that start where a neighbour also starts may still end elsewhere:+ # apple and banana beside avocado share their first letter with it but not+ # their last, and one class at each end separates them.+ if finals:+ candidates.append(f"{prefix}**/*[{finals}]{suffix}")+ if initials and finals:+ candidates.append(f"{prefix}**/[{initials}]*[{finals}]{suffix}")+ # A hand-written mapping may name files no glob can gather. Each one still+ # names itself, so there is always something to fall back to that reads only+ # what the card declared.+ candidates += sorted(wanted)+ for candidate in candidates:+ if set(_files_under_patterns([candidate], data_files)) == wanted:+ return candidate+ # Nothing names these files and only these files: train aa and bb beside+ # test ab and ba defeat every class a glob can carry. Against a card, that+ # is refused rather than read wide or short; against a guess from the file+ # names, the folder is still covered.+ return f"{base}/*{suffix}" if fallback_glob else ""def _resolve_seed_hf_path(dataset_name: str,data_files: list[str],split: str = DEFAULT_SPLIT,+ subset: str | None = None,+ configs: list[dict[str, Any]] | None = None,+ repo_files: list[str] | None = None,) -> str | None:- selected = _select_best_file(data_files, split)+ declared = _declared_split_patterns(configs or [], split, subset)+ declared_files = _files_under_patterns(declared, data_files, as_the_loader_would = True)+ # The card's own mapping beats any guess from the folder names, as long as it+ # resolves to files that are really there.+ if declared_files:+ suffix = _dominant_suffix(declared_files)+ declared_files = _of_suffix(declared_files, suffix)+ exact = suffix+ suffix = _extension_glob(declared_files, suffix, repo_files or data_files)+ if suffix == exact and len({Path(f).suffix.lower() for f in declared_files}) > 1:+ # The card put both of the builder's extensions in this split and no+ # glob can take the two without also taking the file that made the+ # combined form unsafe. Refused, rather than read short.+ return None+ pattern = ""+ if len(declared) == 1:+ pattern = _with_data_extension(declared[0], suffix)+ # A split spread over several declared globs cannot be written as one, and+ # neither can a rewritten pattern that takes anything other than what the+ # card declared: the reader skips no hidden folder the loader would, so+ # the rewrite has to exclude one itself.+ if not pattern or set(declared_files) != set(_files_under_patterns([pattern], data_files)):+ pattern = _widened_declared_pattern(declared_files, data_files, split.lower(), suffix)+ # An unrepresentable mapping is refused, not approximated: the endpoint+ # answers 422 and the recipe is never pointed at the wrong rows.+ return f"datasets/{dataset_name}/{pattern}" if pattern else None++ # Without a card the subset is only a label, so narrowing first keeps the+ # pattern from widening back over another config.+ folder = _config_scope(configs or [], subset)+ scoped = _in_subset(+ [f for f in data_files if f.startswith(f"{folder}/")] or data_files+ if folder+ else data_files,+ subset,+ split.lower(),+ )+ split_lower = split.lower()+ labels = _split_labels(split_lower)+ # Sharded names come first and stop the search, whatever else is labelled.+ staged = _staged_split_files(scoped, labels, folder)+ selected = _select_best_file(sorted(staged) or scoped, split)if not selected:return None- ext = Path(selected).suffix.lower()+ suffix = Path(selected).suffix+ ext = suffix.lower()if ext not in DATA_EXTS:return f"datasets/{dataset_name}/{selected}"parent = Path(selected).parent.as_posix()- if not parent or parent == ".":- return f"datasets/{dataset_name}/**/*{ext}"- return f"datasets/{dataset_name}/{parent}/**/*{ext}"+ base = f"datasets/{dataset_name}"+ if parent and parent != ".":+ base = f"{base}/{parent}"++ if _split_folder(selected.lower(), labels):+ # train_a and train_b are one split, so rooting at either drops half.+ folders = staged or {f for f in scoped if _split_rank(f, split_lower) == 0}+ root = _common_parent(sorted(folders | {selected}))+ if root != parent:+ for label in labels:+ # The four shapes `local_options._DIR_NAME_KEYWORD_PATTERNS`+ # accepts, the last for a label between separators, a_train_x.+ for shape in (+ f"**/{label}/**/*{ext}",+ f"**/{label}{_SEP}*/**/*{ext}",+ f"**/*{_SEP}{label}/**/*{ext}",+ f"**/*{_SEP}{label}{_SEP}*/**/*{ext}",+ # train/ beside sets/train_a/ needs a looser form, still+ # only taken where the listing bears it out.+ f"**/{label}*/**/*{ext}",+ f"**/*{label}*/**/*{ext}",+ ):+ if _pattern_fits_the_split(data_files, folders, root, shape):+ return f"{_anchor(dataset_name, root)}/{shape}"+ else:+ # The files this request is for: in the chosen subset, and of this split.+ wanted = staged or {f for f in scoped if _split_rank(f, split_lower) <= 1}+ # A split sharded over sibling folders is anchored at the folder they+ # share: without a card the loader reads those as one split, and a+ # requested subset has already narrowed `scoped`.+ spread = any(Path(f).parent.as_posix() != parent for f in wanted)+ root = _common_parent(sorted(wanted | {selected})) if spread else parent+ stem = Path(selected).name[: -len(suffix)]+ for candidate in _candidate_patterns(stem, suffix, split_lower):+ pattern = f"**/{candidate}" if spread else candidate+ if _pattern_fits_the_split(data_files, wanted, root, pattern):+ return f"{_anchor(dataset_name, root)}/{pattern}"+ # dev-0 beside validation-0 is one split, and no pattern built from the+ # chosen file gathers the other, so the declared-mapping widening is+ # reused. Only where every neighbour is labelled: an unlabelled file may+ # belong to this split, and only the broad glob would keep it.+ prefix = f"{root}/" if root and root != "." else ""+ neighbours = [f for f in data_files if f.startswith(prefix) and f not in wanted]+ if len(wanted) > 1 and all(_carries_another_split(f, split_lower) for f in neighbours):+ widened = _widened_declared_pattern(+ sorted(wanted), data_files, split_lower, suffix, fallback_glob = True+ )+ return f"datasets/{dataset_name}/{widened}"+ return f"{base}/**/*{ext}"def _build_stream_load_kwargs(@@ -356,9 +1055,25 @@ def inspect_seed_dataset() from excpreview_rows: list[dict[str, Any]] = []- data_files = _list_hf_data_files(dataset_name = dataset_name, token = token)-- selected_file = _select_best_file(data_files, split)+ repo_files = _list_hf_repo_files(dataset_name = dataset_name, token = token)+ data_files = [file for file in repo_files if file.lower().endswith(DATA_EXTS)]+ configs = _list_hf_dataset_configs(dataset_name = dataset_name, token = token)++ # Preview the same files the recipe will read, so the rows on screen are not+ # from a config the resolved path excludes.+ declared_files = _files_under_patterns(+ _declared_split_patterns(configs, split, subset),+ data_files,+ as_the_loader_would = True,+ )+ # Same format the resolved path will read, or the rows on screen come from a+ # file the recipe never opens.+ declared_files = _of_suffix(declared_files, _dominant_suffix(declared_files))+ # A config that only names a folder declares no files, and its name need not+ # be that folder, so the fallback previews the folder rather than the repo.+ folder = _config_scope(configs, subset)+ in_folder = [f for f in data_files if f.startswith(f"{folder}/")] if folder else []+ selected_file = _select_best_file(declared_files or in_folder or data_files, split, subset)if selected_file:try:single_file_kwargs = _build_stream_load_kwargs(@@ -406,7 +1121,9 @@ def inspect_seed_dataset(if not data_files:resolved_path = f"datasets/{dataset_name}/**/*.parquet"else:- resolved_path = _resolve_seed_hf_path(dataset_name, data_files, split)+ resolved_path = _resolve_seed_hf_path(+ dataset_name, data_files, split, subset, configs, repo_files+ )if not resolved_path:raise HTTPException(status_code = 422, detail = "unable to resolve seed dataset path")diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.pyindex 6a4d32b470c..94680698505 100644--- a/studio/backend/tests/test_data_recipe_seed.py+++ b/studio/backend/tests/test_data_recipe_seed.py@@ -432,6 +432,1128 @@ def test_a_backend_executed_seed_resolves_the_endpoint_on_the_backend(monkeypatc_resolve_seed_endpoint({"seed_config": "nope"})+_GSM8K_FILES = [+ "main/test-00000-of-00001.parquet",+ "main/train-00000-of-00001.parquet",+ "socratic/test-00000-of-00001.parquet",+ "socratic/train-00000-of-00001.parquet",+]+++ ("files", "split", "subset", "expected"),+ [+ # No card: the loader reads every train-named file as one split.+ (_GSM8K_FILES, "train", None, "datasets/org/repo/**/train-*.parquet"),+ (_GSM8K_FILES, "test", "main", "datasets/org/repo/main/test-*.parquet"),+ (_GSM8K_FILES, "train", "socratic", "datasets/org/repo/socratic/train-*.parquet"),+ (_GSM8K_FILES, "train", "default", "datasets/org/repo/**/train-*.parquet"),+ (+ ["data/test-00000-of-00001.parquet", "data/train-00000-of-00002.parquet"],+ "train",+ "default",+ "datasets/org/repo/data/train-*.parquet",+ ),+ (["test.csv", "train.csv"], "train", None, "datasets/org/repo/train*.csv"),+ (["Train_0.jsonl", "Test_0.jsonl"], "train", None, "datasets/org/repo/Train_*.jsonl"),+ (+ ["raw/gsm_test.jsonl", "raw/gsm_train.jsonl"],+ "train",+ None,+ "datasets/org/repo/raw/gsm_train*.jsonl",+ ),+ # A split named by the file itself can still be sharded, so the pattern has+ # to reach the siblings instead of pinning the first shard.+ (+ ["train.jsonl", "train_2.jsonl", "test.jsonl"],+ "train",+ None,+ "datasets/org/repo/train*.jsonl",+ ),+ (+ ["raw/gsm_test.jsonl", "raw/gsm_train.jsonl", "raw/gsm_train_2.jsonl"],+ "train",+ None,+ "datasets/org/repo/raw/gsm_train*.jsonl",+ ),+ (+ ["en/test/0000.parquet", "en/train/0000.parquet", "fr/train/0000.parquet"],+ "train",+ "fr",+ "datasets/org/repo/fr/train/**/*.parquet",+ ),+ (["data/part-0.parquet"], "train", "main", "datasets/org/repo/data/**/*.parquet"),+ ],+)+def test_seed_hf_path_keeps_the_split_and_subset(+ monkeypatch, tmp_path, files, split, subset, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ assert seed_route._resolve_seed_hf_path("org/repo", files, split, subset) == expected+++def test_seed_preview_file_comes_from_the_chosen_subset(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ assert (+ seed_route._select_best_file(_GSM8K_FILES, "train", "socratic")+ == "socratic/train-00000-of-00001.parquet"+ )+++# fineweb-edu: the config name is not a folder, so a folder-name guess reads a+# different config entirely.+_SAMPLE_FILES = [+ "data/CC-MAIN-2013-20/train-00000-of-00014.parquet",+ "sample/10BT/000_00000.parquet",+ "sample/10BT/001_00000.parquet",+]+_SAMPLE_CONFIGS = [+ {"config_name": "default", "data_files": [{"split": "train", "path": "data/*/*"}]},+ {"config_name": "sample-10BT", "data_files": [{"split": "train", "path": "sample/10BT/*"}]},+]+++ ("subset", "expected"),+ [+ ("sample-10BT", "datasets/org/repo/sample/10BT/*.parquet"),+ (None, "datasets/org/repo/data/*/*.parquet"),+ ],+)+def test_seed_hf_path_follows_the_dataset_card_config_mapping(+ monkeypatch, tmp_path, subset, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ resolved = seed_route._resolve_seed_hf_path(+ "org/repo", _SAMPLE_FILES, "train", subset, _SAMPLE_CONFIGS+ )+ assert resolved == expected+++def test_seed_hf_path_ignores_a_card_mapping_pointing_at_nothing(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [{"config_name": "main", "data_files": [{"split": "train", "path": "gone/*"}]}]+ resolved = seed_route._resolve_seed_hf_path("org/repo", _GSM8K_FILES, "train", "main", configs)+ assert resolved == "datasets/org/repo/main/train-*.parquet"+++ "data_files",+ [+ "mapped/train-00000.parquet",+ ["mapped/train-00000.parquet"],+ {"train": "mapped/train-*.parquet"},+ [{"split": "train", "path": "mapped/train-*.parquet"}],+ ],+)+def test_seed_hf_path_reads_every_card_data_files_shape(monkeypatch, tmp_path, data_files):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["mapped/train-00000.parquet", "other/train-00000.parquet"]+ configs = [{"config_name": "unrelated", "data_files": data_files}]+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", "unrelated", configs)+ assert resolved.startswith("datasets/org/repo/mapped/")+++ ("configs", "expected"),+ [+ # A card with one config uses it whatever it is called.+ (+ [+ {+ "config_name": "plain_text",+ "data_files": [{"split": "train", "path": "pt/train-*"}],+ }+ ],+ "datasets/org/repo/pt/train-*.parquet",+ ),+ # Another config can be flagged as the default one.+ (+ [+ {"config_name": "a", "data_files": [{"split": "train", "path": "a/train-*"}]},+ {+ "config_name": "b",+ "default": True,+ "data_files": [{"split": "train", "path": "pt/train-*"}],+ },+ ],+ "datasets/org/repo/pt/train-*.parquet",+ ),+ ],+)+def test_seed_hf_path_finds_the_default_config_without_a_subset(+ monkeypatch, tmp_path, configs, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["pt/train-0.parquet", "a/train-0.parquet"]+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) == expected+++def test_seed_card_globs_keep_their_character_classes(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/train-2-of-9.parquet", "data/train-7-of-9.parquet"]+ assert seed_route._files_under_patterns(["data/train-[0-4]*.parquet"], files) == [+ "data/train-2-of-9.parquet"+ ]+ assert seed_route._files_under_patterns(["data/[!x]*.parquet"], files) == files+ # An unclosed bracket is a literal, not a syntax error.+ assert seed_route._files_under_patterns(["a[b.parquet"], ["a[b.parquet"]) == ["a[b.parquet"]+++def test_seed_hf_path_covers_a_split_declared_across_two_folders(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["sets/a/train-0.parquet", "sets/b/train-0.parquet", "other/test-0.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": ["sets/a/train-*.parquet", "sets/b/train-*.parquet"]}+ ],+ }+ ]+ # One glob cannot name two folders, so cover the folder holding both rather+ # than dropping one of them, keeping the split in the pattern.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/sets/**/*train*.parquet"+ )+++_LABELLED_FILES = [+ "main-train.parquet",+ "main-test.parquet",+ "socratic-train.parquet",+ "socratic-test.parquet",+]+++ ("subset", "expected"),+ [+ ("socratic", "datasets/org/repo/socratic-train*.parquet"),+ ("main", "datasets/org/repo/main-train*.parquet"),+ ],+)+def test_seed_hf_path_reads_a_subset_labelled_in_the_file_name(+ monkeypatch, tmp_path, subset, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ resolved = seed_route._resolve_seed_hf_path("org/repo", _LABELLED_FILES, "train", subset)+ assert resolved == expected+++ ("files", "split", "expected"),+ [+ (["data/dev.jsonl", "data/train.jsonl"], "validation", "datasets/org/repo/data/dev*.jsonl"),+ (+ ["data/valid-0.parquet", "data/train-0.parquet"],+ "validation",+ "datasets/org/repo/data/valid-*.parquet",+ ),+ (+ ["data/training-0.parquet", "data/test-0.parquet"],+ "train",+ "datasets/org/repo/data/*train*.parquet",+ ),+ ],+)+def test_seed_hf_path_follows_the_split_aliases_datasets_uses(+ monkeypatch, tmp_path, files, split, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ assert seed_route._resolve_seed_hf_path("org/repo", files, split) == expected+++ ("files", "expected"),+ [+ # A folder qualified by a separator is the split folder too.+ (["train_a/0.parquet", "test/0.parquet"], "datasets/org/repo/train_a/**/*.parquet"),+ (+ ["data/train/0.parquet", "data/test/0.parquet"],+ "datasets/org/repo/data/train/**/*.parquet",+ ),+ # "pretrain" is a different word, so train/ still wins.+ (["pretrain/0.parquet", "train/0.parquet"], "datasets/org/repo/train/**/*.parquet"),+ ],+)+def test_seed_hf_path_reads_a_qualified_split_folder(monkeypatch, tmp_path, files, expected):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train") == expected+++def test_seed_hf_path_widens_through_an_alias_named_split(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["sets/a/dev-0.parquet", "sets/b/dev-0.parquet", "sets/a/train-0.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "validation", "path": ["sets/a/dev-*.parquet", "sets/b/dev-*.parquet"]},+ {"split": "train", "path": "sets/a/train-*.parquet"},+ ],+ }+ ]+ # The files say dev, not validation, so the widened form has to say dev too+ # or it takes the train shard with it.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "validation", None, configs)+ == "datasets/org/repo/sets/**/*dev*.parquet"+ )+++def test_seed_hf_path_counts_a_digit_as_a_label_separator(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ # The loader's own separators include digits, so train1 is train.+ files = ["data/train-0.parquet", "data/train1.parquet", "data/test-0.parquet"]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train")+ == "datasets/org/repo/data/*train*.parquet"+ )+ assert (+ seed_route._resolve_seed_hf_path(+ "org/repo", ["data/train-0.parquet", "data/testing-0.parquet"], "train"+ )+ == "datasets/org/repo/data/train-*.parquet"+ )+++def test_seed_hf_path_keeps_a_split_spread_over_sibling_folders(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["sets/a/part.parquet", "sets/b/part.parquet", "sets/c/test.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": ["sets/a/part.parquet", "sets/b/part.parquet"]},+ {"split": "test", "path": "sets/c/test.parquet"},+ ],+ }+ ]+ # Nothing names the split, but the two declared files share a name the test+ # file does not, so the union stays off sets/c.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/sets/**/part*.parquet"+ )+++def test_seed_hf_path_unions_declared_files_with_unrelated_names(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["sets/a/part-a.parquet", "sets/b/chunk-b.parquet", "sets/c/test.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": ["sets/a/part-a.parquet", "sets/b/chunk-b.parquet"]},+ {"split": "test", "path": "sets/c/test.parquet"},+ ],+ }+ ]+ # Nothing is shared but the first letters, and a class is the only union the+ # reader understands: it rejects {a,b} outright.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/sets/**/[cp]*.parquet"+ )+++def test_seed_format_inference_stops_where_the_loader_stops(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ # 200 csv sort ahead of 201 parquet, and the loader only looks at the first+ # 200, so it builds csv and the recipe has to agree.+ files = [f"data/{i:03d}.csv" for i in range(200)] + [+ f"data/z{i:03d}.parquet" for i in range(201)+ ]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/data/*.csv"+ )+++ ("files", "expected"),+ [+ # Most files wins, so the long csv name does not decide the format.+ (+ ["data/a-very-long-name.csv", "data/z.parquet", "data/y.parquet"],+ "datasets/org/repo/data/*.parquet",+ ),+ (["data/a.csv", "data/z.parquet"], "datasets/org/repo/data/*.parquet"),+ (["data/a.csv", "data/b.csv"], "datasets/org/repo/data/*.csv"),+ ],+)+def test_seed_hf_path_picks_the_format_the_loader_would_build(+ monkeypatch, tmp_path, files, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) == expected+++def test_seed_hf_path_combines_bare_and_explicit_train_entries(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [+ {+ "config_name": "default",+ "data_files": [+ "data/part-a.parquet",+ {"split": "train", "path": "data/part-b.parquet"},+ ],+ }+ ]+ assert seed_route._declared_split_patterns(configs, "train") == [+ "data/part-a.parquet",+ "data/part-b.parquet",+ ]+ files = ["data/part-a.parquet", "data/part-b.parquet", "data/test-0.parquet"]+ # Neither declared file names the split, but they share a name the test file+ # does not, so the union stays exact.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/data/part-*.parquet"+ )+++def test_seed_hf_path_keeps_a_folder_subset_whose_files_are_generic(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ # en/ is the config even though its file says nothing about the split, so+ # other/train.parquet must not win it.+ files = ["en/data.parquet", "other/train.parquet"]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "en")+ == "datasets/org/repo/en/**/*.parquet"+ )+++ ("subset", "expected"),+ [+ ("main", "datasets/org/repo/train-main*.parquet"),+ ("socratic", "datasets/org/repo/train-socratic*.parquet"),+ ],+)+def test_seed_hf_path_keeps_a_subset_written_after_the_split(+ monkeypatch, tmp_path, subset, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["train-main.parquet", "test-main.parquet", "train-socratic.parquet"]+ # train-*.parquet fits the subset slice but takes the other subset with it,+ # so the candidate has to be judged against the whole listing.+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset) == expected+++ ("subset", "expected"),+ [+ ("french", "datasets/org/repo/fr/train*.parquet"),+ ("english", "datasets/org/repo/en/train*.parquet"),+ ],+)+def test_seed_hf_path_scopes_a_config_to_its_data_dir(monkeypatch, tmp_path, subset, expected):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [+ {"config_name": "english", "data_dir": "en"},+ {"config_name": "french", "data_dir": "fr"},+ ]+ files = ["en/train.parquet", "en/test.parquet", "fr/train.parquet", "fr/test.parquet"]+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset, configs) == expected+++def test_seed_hf_path_reads_data_files_relative_to_data_dir(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [+ {+ "config_name": "a",+ "data_dir": "d",+ "data_files": [{"split": "train", "path": "train-*.parquet"}],+ }+ ]+ files = ["d/train-0.parquet", "d/test-0.parquet"]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "a", configs)+ == "datasets/org/repo/d/train-*.parquet"+ )+++def test_seed_format_vote_ignores_folder_metadata(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]+ files = ["data/metadata.csv", "data/metadata2.csv", "data/shard.parquet"]+ # metadata.csv never decides a builder for the loader, so it cannot here.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/data/*.parquet"+ )+++ ("subset", "expected"),+ [+ ("Foo", "datasets/org/repo/F/train.parquet"),+ ("foo", "datasets/org/repo/f/train.parquet"),+ ],+)+def test_seed_hf_path_matches_config_names_case_sensitively(+ monkeypatch, tmp_path, subset, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [+ {"config_name": "Foo", "data_files": [{"split": "train", "path": "F/train.parquet"}]},+ {"config_name": "foo", "data_files": [{"split": "train", "path": "f/train.parquet"}]},+ ]+ files = ["F/train.parquet", "f/train.parquet"]+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset, configs) == expected+++def test_seed_hf_path_keeps_a_hand_listed_split_off_its_neighbours(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/apple.parquet", "data/banana.parquet", "data/avocado.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": ["data/apple.parquet", "data/banana.parquet"]},+ {"split": "test", "path": "data/avocado.parquet"},+ ],+ }+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["data/apple.parquet", "data/banana.parquet"]+++def test_seed_hf_path_never_widens_a_declared_split_over_another(monkeypatch, tmp_path):+ """No glob names these two alone, so the recipe reads what it can name."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/ax.parquet", "data/bx.parquet", "data/cx.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": ["data/ax.parquet", "data/cx.parquet"]},+ {"split": "test", "path": "data/bx.parquet"},+ ],+ }+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["data/ax.parquet", "data/cx.parquet"]+++def test_seed_hf_path_keeps_shards_whose_folders_also_hold_other_splits(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [+ "a/train-0.parquet",+ "a/test-0.parquet",+ "b/train-1.parquet",+ "b/test-1.parquet",+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["a/train-0.parquet", "b/train-1.parquet"]+++def test_seed_hf_path_refuses_a_mixed_json_split_it_cannot_name(monkeypatch, tmp_path):+ """*.json* is run over the whole repo, so a a.json.gz would be read too."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/a.json", "data/b.jsonl"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]++ resolved = seed_route._resolve_seed_hf_path(+ "org/repo", files, "train", None, configs, [*files, "data/a.json.gz"]+ )++ assert resolved is None+++def test_seed_hf_path_keeps_a_trailing_globstar_a_whole_component(monkeypatch, tmp_path):+ """fsspec refuses data/**.parquet outright, so the card's data/** grows a /*."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/a/train.parquet", "data/train.parquet"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/**"}]}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ assert resolved == "datasets/org/repo/data/**/*.parquet"+++def test_seed_hf_path_leaves_hidden_folders_out_of_a_broad_card_glob(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/train.parquet", ".backup/train.parquet"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "**/*"}]}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert matched == ["data/train.parquet"]+++def test_seed_hf_path_reads_a_hidden_folder_the_card_names(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/train.parquet", ".backup/train.parquet"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": ".backup/*"}]}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert matched == [".backup/train.parquet"]+++def test_seed_hf_path_keeps_both_extensions_of_one_builder(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/a.json", "data/b.jsonl"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ assert resolved == "datasets/org/repo/data/*.json*"+++def test_seed_hf_path_finds_sharded_names_under_a_config_data_dir(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [+ "scope/data/train-00000-of-00001.parquet",+ "scope/data/test-00000-of-00001.parquet",+ "scope/extras/train-extra.parquet",+ ]+ configs = [{"config_name": "scoped", "data_dir": "scope"}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", "scoped", configs)++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert matched == ["scope/data/train-00000-of-00001.parquet"]+++def test_seed_hf_path_refuses_a_declared_split_no_glob_can_name(monkeypatch, tmp_path):+ """aa and bb cannot be told from ab and ba by any class a glob can carry."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/aa.parquet", "data/ab.parquet", "data/ba.parquet", "data/bb.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": ["data/aa.parquet", "data/bb.parquet"]},+ {"split": "test", "path": ["data/ab.parquet", "data/ba.parquet"]},+ ],+ }+ ]++ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) is None+++def test_seed_hf_path_stops_at_the_sharded_names_the_loader_reads_first(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [+ "data/train-00000-of-00001.parquet",+ "data/test-00000-of-00001.parquet",+ "extras/train-extra.parquet",+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert matched == ["data/train-00000-of-00001.parquet"]+++ "configs",+ [+ [+ {+ "config_name": "default",+ "data_dir": "./data",+ "data_files": [+ {"split": "train", "path": "a.parquet"},+ {"split": "test", "path": "b.parquet"},+ ],+ }+ ],+ [+ {+ "config_name": "default",+ "data_dir": ".",+ "data_files": [+ {"split": "train", "path": "./data/a.parquet"},+ {"split": "test", "path": "./data/b.parquet"},+ ],+ }+ ],+ ],+)+def test_seed_hf_path_reads_a_card_that_spells_paths_with_a_dot(monkeypatch, tmp_path, configs):+ """The listing has no ./ in it, so a card carrying one has to lose it."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/a.parquet", "data/b.parquet"]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ assert resolved == "datasets/org/repo/data/a.parquet"+++ ("split", "expected"),+ [("Train", "datasets/org/repo/b.parquet"), ("train", "datasets/org/repo/a.parquet")],+)+def test_seed_hf_path_reads_a_split_name_exactly(monkeypatch, tmp_path, split, expected):+ """load_dataset is handed the name the caller asked for, not a folded one."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["a.parquet", "b.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": "a.parquet"},+ {"split": "Train", "path": "b.parquet"},+ ],+ }+ ]++ assert seed_route._resolve_seed_hf_path("org/repo", files, split, None, configs) == expected+++def test_seed_hf_path_reads_declared_files_under_data_dir_even_when_they_repeat_it(+ monkeypatch, tmp_path+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/data/train.parquet", "data/train.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_dir": "data",+ "data_files": [{"split": "train", "path": "data/train.parquet"}],+ }+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ assert resolved == "datasets/org/repo/data/data/train.parquet"+++def test_seed_hf_path_collapses_a_parent_component_in_a_card_path(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/a.parquet", "data/b.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_dir": "staging/../data",+ "data_files": [+ {"split": "train", "path": "a.parquet"},+ {"split": "test", "path": "b.parquet"},+ ],+ }+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ assert resolved == "datasets/org/repo/data/a.parquet"+++def test_seed_hf_path_leaves_loader_ignored_metadata_out_of_a_card_glob(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["train.jsonl", "dataset_info.json"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "*"}]}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert matched == ["train.jsonl"]+++ ("pattern", "expected"),+ [("[^a].parquet", ["a.parquet"]), ("[!a].parquet", ["b.parquet"])],+)+def test_seed_glob_reads_a_caret_in_a_class_literally(monkeypatch, tmp_path, pattern, expected):+ """fsspec negates on ! alone, so [^a] is the two characters ^ and a."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)++ assert seed_route._files_under_patterns([pattern], ["a.parquet", "b.parquet"]) == expected+++def test_seed_hf_path_keeps_a_dunder_file_a_card_declared(monkeypatch, tmp_path):+ """The loader skips a __folder, not a file whose name happens to start that way."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/__train.json", "data/train.json"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["data/__train.json", "data/train.json"]+++def test_seed_config_named_Default_is_not_the_implicit_default(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)++ assert (+ seed_route._pick_config([{"config_name": "Default"}, {"config_name": "other"}], None)+ is None+ )+++def test_seed_hf_path_ignores_an_extension_collision_out_of_reach(monkeypatch, tmp_path):+ """The glob is anchored at data/, so a name in other/ cannot spoil it."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/a.json", "data/b.jsonl"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]++ resolved = seed_route._resolve_seed_hf_path(+ "org/repo", files, "train", None, configs, [*files, "other/c.json.gz"]+ )++ assert resolved == "datasets/org/repo/data/*.json*"+++ ("subset", "expected"),+ [+ ("Foo", "datasets/org/repo/Foo/train*.parquet"),+ ("foo", "datasets/org/repo/foo/train*.parquet"),+ ],+)+def test_seed_hf_path_narrows_to_the_subset_folder_by_its_own_case(+ monkeypatch, tmp_path, subset, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["Foo/train.parquet", "foo/train.parquet"]++ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset) == expected+++def test_seed_format_vote_counts_every_suffix_in_a_name(monkeypatch, tmp_path):+ """a.csv.parquet is a csv to the loader as much as a parquet."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)++ assert seed_route._dominant_suffix(["data/a.csv.parquet", "data/b.csv"]) == ".csv"+ assert seed_route._dominant_suffix(["a.parquet", "b.parquet", "c.json"]) == ".parquet"+++def test_seed_format_vote_takes_the_window_before_dropping_metadata(monkeypatch, tmp_path):+ """A shard past the window is one the loader never sees either."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [f"a{i:04d}/metadata.csv" for i in range(200)] + ["z/shard.parquet"]++ assert seed_route._dominant_suffix(files) == ".csv"+ assert seed_route._dominant_suffix(["a/metadata.csv", "b/shard.parquet"]) == ".parquet"+++def test_seed_hf_path_unions_an_exact_and_a_qualified_split_folder(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["train/0.parquet", "sets/train_a/1.parquet", "sets/test_b/2.parquet"]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["sets/train_a/1.parquet", "train/0.parquet"]+++def test_seed_hf_path_keeps_folders_naming_the_split_in_the_middle(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [+ "sets/a_train_x/0.parquet",+ "sets/b_train_y/0.parquet",+ "sets/c_test_z/0.parquet",+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["sets/a_train_x/0.parquet", "sets/b_train_y/0.parquet"]+++def test_seed_hf_path_keeps_qualified_split_folders_together(monkeypatch, tmp_path):+ """train_a and train_b are both train to the loader, so both must be read."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [+ "sets/train_a/0.parquet",+ "sets/train_b/1.parquet",+ "sets/test_a/2.parquet",+ ]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["sets/train_a/0.parquet", "sets/train_b/1.parquet"]+++def test_seed_hf_path_gathers_a_split_written_under_two_aliases(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/dev-0.parquet", "data/validation-0.parquet", "data/test-0.parquet"]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "validation")++ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)+ assert sorted(matched) == ["data/dev-0.parquet", "data/validation-0.parquet"]+++def test_seed_hf_path_counts_json_and_jsonl_as_two_extensions(monkeypatch, tmp_path):+ """`load.infer_module_for_data_files_list` votes per extension, parquet on ties."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/a.json", "data/b.jsonl", "data/c.parquet"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)++ assert resolved == "datasets/org/repo/data/*.parquet"+++def test_seed_hf_path_keeps_shards_spread_over_sibling_folders(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["a/train-0.parquet", "b/train-1.parquet", "a/test-0.parquet"]++ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")++ assert resolved == "datasets/org/repo/**/train-*.parquet"+++def test_seed_hf_path_reads_sibling_folders_as_one_split_without_a_card(monkeypatch, tmp_path):+ """Without a card there are no configs, so every train file is the train split."""+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [+ "main/train-0.parquet",+ "main/test-0.parquet",+ "socratic/train-0.parquet",+ "socratic/test-0.parquet",+ ]++ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train")+ == "datasets/org/repo/**/train-*.parquet"+ )+ # Ask for one of them and only that one is read.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "socratic")+ == "datasets/org/repo/socratic/train-*.parquet"+ )+++def test_seed_preview_file_follows_the_config_data_dir(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["en/train.parquet", "fr/train.parquet"]+ configs = [{"config_name": "french", "data_dir": "fr"}]+ monkeypatch.setattr(seed_route, "_list_hf_repo_files", lambda **kwargs: files)+ monkeypatch.setattr(seed_route, "_list_hf_dataset_configs", lambda **kwargs: configs)+ monkeypatch.setattr(+ seed_route, "refuse_unauthorized_dataset_preview", lambda *args, **kwargs: None+ )+ seen: list[str | None] = []++ def fake_preview(*, load_dataset_fn, load_kwargs, preview_size):+ seen.append((load_kwargs.get("data_files") or [None])[0])+ return [{"text": "row"}]++ monkeypatch.setattr(seed_route, "_load_preview_rows", fake_preview)++ response = seed_route.inspect_seed_dataset(+ SimpleNamespace(+ dataset_name = "org/repo",+ split = "train",+ subset = "french",+ hf_token = None,+ preview_size = 1,+ ),+ allow_ambient_token = False,+ )++ assert seen[0] == "fr/train.parquet"+ assert response.resolved_path == "datasets/org/repo/fr/train*.parquet"+++def test_seed_hf_path_treats_an_unnamed_config_as_the_default(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ configs = [+ {+ "data_files": [+ {"split": "train", "path": "a/part.parquet"},+ {"split": "test", "path": "b/part.parquet"},+ ]+ }+ ]+ files = ["a/part.parquet", "b/part.parquet"]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "test", "default", configs)+ == "datasets/org/repo/b/part.parquet"+ )+++def test_seed_hf_path_ignores_a_subset_label_that_is_not_the_config(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ # x/main.parquet carries the label but nothing of the split, so it is a name+ # collision rather than the config.+ files = ["x/main.parquet", "y/train-0.parquet"]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "main")+ == "datasets/org/repo/y/train-*.parquet"+ )+++def test_seed_hf_path_keeps_the_split_when_widening_several_declared_globs(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["sets/a/train-0.parquet", "sets/b/train-0.parquet", "sets/a/test-0.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ # Declared twice for the same split, which must not lose the second.+ {"split": "train", "path": "sets/a/train-*"},+ {"split": "train", "path": "sets/b/train-*"},+ ],+ }+ ]+ assert seed_route._declared_split_patterns(configs, "train") == [+ "sets/a/train-*",+ "sets/b/train-*",+ ]+ # sets/**/*.parquet would take the test file back; the split-named form does not.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/sets/**/*train*.parquet"+ )+++def test_seed_hf_path_keeps_a_split_whose_name_carries_a_separator(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = [+ "data/validation_matched-000.parquet",+ "data/validation_mismatched-000.parquet",+ "data/train-000.parquet",+ ]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "validation_matched")+ == "datasets/org/repo/data/validation_matched-*.parquet"+ )+++def test_seed_hf_path_keeps_a_subset_whose_name_carries_a_separator(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["sample-10BT-train.parquet", "sample-100BT-train.parquet"]+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "sample-10BT")+ == "datasets/org/repo/sample-10BT-train*.parquet"+ )+++def test_seed_hf_path_widens_through_split_named_folders(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["sets/train_a/0.parquet", "sets/train_b/0.parquet", "sets/test/0.parquet"]+ configs = [+ {+ "config_name": "default",+ "data_files": [+ {"split": "train", "path": ["sets/train_a/*", "sets/train_b/*"]},+ {"split": "test", "path": "sets/test/*"},+ ],+ }+ ]+ # The shard names say nothing, so the split has to come from the folders.+ assert (+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)+ == "datasets/org/repo/sets/**/*train*/**/*.parquet"+ )+++def test_seed_globstar_keeps_the_folder_boundary(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/train.csv", "data/z/train.csv", "data/nottrain.parquet"]+ assert seed_route._files_under_patterns(["data/**/train.*"], files) == [+ "data/train.csv",+ "data/z/train.csv",+ ]+++def test_seed_declared_files_match_the_glob_not_its_prefix(monkeypatch, tmp_path):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/train-notes.json", "data/train-00000-of-00002.parquet"]+ assert seed_route._files_under_patterns(["data/train-*.parquet"], files) == [+ "data/train-00000-of-00002.parquet"+ ]+ assert seed_route._files_under_patterns(["data/*/x.parquet"], ["data/a/b/x.parquet"]) == []+ assert seed_route._files_under_patterns(["data/**/x.parquet"], ["data/a/b/x.parquet"]) == [+ "data/a/b/x.parquet"+ ]+++ ("files", "expected"),+ [+ # train-part.parquet wins on length, but questions_train.parquet is train+ # too, so the pattern has to reach both without taking test-part.+ (+ ["data/train-part.parquet", "data/questions_train.parquet", "data/test-part.parquet"],+ "datasets/org/repo/data/*train*.parquet",+ ),+ # A split named mid-name, the conventional sharded form.+ (+ ["data/questions_train_000.jsonl", "data/questions_test_000.jsonl"],+ "datasets/org/repo/data/*train*.jsonl",+ ),+ # A dotted split name: only the final extension comes off the stem.+ (+ ["data/questions.train.parquet", "data/questions.test.parquet"],+ "datasets/org/repo/data/*train*.parquet",+ ),+ # "training" is datasets' own alias for train, so both belong to it.+ (+ ["data/train-0.parquet", "data/training-0.parquet", "data/test-0.parquet"],+ "datasets/org/repo/data/*train*.parquet",+ ),+ ],+)+def test_seed_hf_path_reaches_every_file_of_the_split_and_no_other(+ monkeypatch, tmp_path, files, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train") == expected+++ ("declared", "expected"),+ [+ ("data/train.*", "datasets/org/repo/data/train.parquet"),+ ("data/train-*.*", "datasets/org/repo/data/train-*.parquet"),+ ("data/train-*", "datasets/org/repo/data/train-*.parquet"),+ ("data/train-0.parquet", "datasets/org/repo/data/train-0.parquet"),+ ],+)+def test_seed_hf_path_gives_a_card_glob_a_readable_extension(+ monkeypatch, tmp_path, declared, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ files = ["data/train.parquet", "data/train-0.parquet", "data/test-0.parquet"]+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": declared}]}]+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) == expected+++ ("files", "expected"),+ [+ (["part-0.parquet"], "datasets/org/repo/**/*.parquet"),+ (["data/part-0.parquet", "data/part-1.parquet"], "datasets/org/repo/data/**/*.parquet"),+ (+ ["data/test/0.parquet", "data/train/0.parquet"],+ "datasets/org/repo/data/train/**/*.parquet",+ ),+ # Split folders at the repo root: the shorter "test" path used to win on+ # length alone, so a train recipe was served the test split.+ (["test/0.parquet", "train/0.parquet"], "datasets/org/repo/train/**/*.parquet"),+ ],+)+def test_seed_hf_path_still_globs_the_directory_without_a_split_in_the_name(+ monkeypatch, tmp_path, files, expected+):+ seed_route = _load_seed_route(monkeypatch, tmp_path)+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train") == expected++def test_validate_resolves_the_hf_seed_endpoint_like_jobs(monkeypatch):pytest.importorskip("fastapi")backend_root = Path(__file__).resolve().parent.parentdiff --git a/studio/backend/tests/test_hub_token_caller_identity.py b/studio/backend/tests/test_hub_token_caller_identity.pyindex 29be59bca81..a888de1aa96 100644--- a/studio/backend/tests/test_hub_token_caller_identity.py+++ b/studio/backend/tests/test_hub_token_caller_identity.py@@ -834,7 +834,7 @@ def test_seed_inspection_derives_its_policy_from_the_caller(monkeypatch):seen["token"] = tokenreturn []- monkeypatch.setattr(seed_routes, "_list_hf_data_files", _fake_list)+ monkeypatch.setattr(seed_routes, "_list_hf_repo_files", _fake_list)for via_api_key in (True, False):_router_client(seed_routes.router, "/api/data-recipe", via_api_key = via_api_key).post(@@ -852,7 +852,7 @@ def test_an_explicit_seed_token_wins_for_either_caller(monkeypatch):_counting_probe(monkeypatch, True)monkeypatch.setattr(seed_routes,- "_list_hf_data_files",+ "_list_hf_repo_files",lambda *, dataset_name, token: seen.update(token = token) or [],)monkeypatch.setattr(seed_routes, "load_dataset", lambda **_k: iter([]), raising = False)@@ -1362,7 +1362,7 @@ def test_a_seed_preview_off_the_cache_is_refused(monkeypatch, reason, hf_token,def _never(*_a, **_k):raise AssertionError(f"{reason} reached the dataset load")- monkeypatch.setattr(seed_routes, "_list_hf_data_files", _never)+ monkeypatch.setattr(seed_routes, "_list_hf_repo_files", _never)payload = SimpleNamespace(dataset_name = "org/private",