Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: use only the chosen split and subset for Hugging Face recipe data (#11294)

Nilayca89af7a7211
diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py
index 4b9a58c64c7..37c23dae1d7 100644
--- a/studio/backend/routes/data_recipe/seed.py
+++ b/studio/backend/routes/data_recipe/seed.py
@@ -13,7 +13,7 @@ import os
import re
import shutil
from itertools import islice
-from pathlib import Path
+from pathlib import Path, PurePosixPath
from typing import Any
from uuid import uuid4
@@ -22,6 +22,14 @@ from fastapi import APIRouter, Depends, HTTPException, UploadFile, File as FastA
from auth.authentication import allow_ambient_hf_token
from core.data_recipe.jsonable import to_preview_jsonable
+from hub.services.datasets.local_options import (
+ _sharded_splits,
+ _IGNORED_DATA_FILENAMES,
+ _MAX_MODULE_INFERENCE_FILES,
+ _METADATA_FILENAMES,
+ _SEP,
+ _SPLIT_KEYWORDS,
+)
from hub.utils.dataset_cache import refuse_unauthorized_dataset_preview
from hub.utils.hf_tokens import HfTokenArg, hf_token_arg
from loggers import get_logger
@@ -68,6 +76,7 @@ def _chunking() -> Any:
DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv")
DEFAULT_SPLIT = "train"
+DEFAULT_CONFIG = "default"
LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"}
UNSTRUCTURED_ALLOWED_EXTS = {".pdf", ".docx", ".txt", ".md"}
SEED_UPLOAD_DIR = LazyPath(seed_uploads_root)
@@ -101,7 +110,10 @@ def _normalize_optional_text(value: str | None) -> str | None:
return trimmed if trimmed else None
-def _list_hf_data_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:
+def _list_hf_repo_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:
+ """Every file in the repo: the reader globs over all of them, not just the
+ ones a builder can load, so a pattern is judged against the whole listing.
+ """
try:
from huggingface_hub import HfApi
from huggingface_hub.utils import HfHubHTTPError
@@ -109,51 +121,738 @@ def _list_hf_data_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:
return []
try:
api = HfApi(token = token)
- repo_files = api.list_repo_files(dataset_name, repo_type = "dataset", token = token)
- return [file for file in repo_files if file.lower().endswith(DATA_EXTS)]
+ return api.list_repo_files(dataset_name, repo_type = "dataset", token = token)
except (HfHubHTTPError, OSError, ValueError):
return []
-def _select_best_file(data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:
+def _list_hf_data_files(*, dataset_name: str, token: HfTokenArg) -> list[str]:
+ repo_files = _list_hf_repo_files(dataset_name = dataset_name, token = token)
+ return [file for file in repo_files if file.lower().endswith(DATA_EXTS)]
+
+
+def _list_hf_dataset_configs(*, dataset_name: str, token: HfTokenArg) -> list[dict[str, Any]]:
+ """The `configs:` block of the dataset card, which maps a config to its own file globs.
+
+ A config name is not required to be a folder name: fineweb-edu's `sample-10BT`
+ lives under `sample/10BT/`, so a folder-name guess reads a different config.
+ """
+ try:
+ from huggingface_hub import HfApi
+ from huggingface_hub.utils import HfHubHTTPError
+ except ImportError:
+ return []
+ try:
+ api = HfApi(token = token)
+ card_data = api.dataset_info(dataset_name, token = token).card_data
+ configs = (card_data or {}).get("configs") if card_data is not None else None
+ except (HfHubHTTPError, OSError, ValueError, AttributeError):
+ return []
+ return [config for config in (configs or []) if isinstance(config, dict)]
+
+
+def _as_pattern_list(paths: Any) -> list[str]:
+ if isinstance(paths, str):
+ return [paths]
+ if isinstance(paths, list):
+ return [path for path in paths if isinstance(path, str)]
+ return []
+
+
+def _patterns_for_split(
+ data_files: Any,
+ split: str,
+ *,
+ exact: bool = True,
+) -> list[str]:
+ """Every shape a card may use for `data_files`, as hub/services/datasets does.
+
+ A bare string or list of strings is the shorthand for the train split. The
+ name is read exactly first: a card may declare Train beside train, and
+ `load_dataset` is handed the name the caller asked for, not a folded one.
+ """
+ if isinstance(data_files, str):
+ return [data_files] if _same_split(DEFAULT_SPLIT, split, exact) else []
+ if isinstance(data_files, dict):
+ for name, paths in data_files.items():
+ if _same_split(str(name), split, exact):
+ return _as_pattern_list(paths)
+ return []
+ if isinstance(data_files, list):
+ # Entry by entry: a list may declare one split more than once, and
+ # stopping at the first match drops the rest of it.
+ patterns: list[str] = []
+ for entry in data_files:
+ if isinstance(entry, str):
+ if _same_split(DEFAULT_SPLIT, split, exact):
+ patterns.append(entry)
+ elif isinstance(entry, dict) and _same_split(
+ str(entry.get("split") or ""), split, exact
+ ):
+ patterns.extend(_as_pattern_list(entry.get("path")))
+ return patterns
+ return []
+
+
+def _same_split(declared: str, split: str, exact: bool) -> bool:
+ return declared == split if exact else declared.lower() == split.lower()
+
+
+def _declared_split_patterns(
+ configs: list[dict[str, Any]],
+ split: str = DEFAULT_SPLIT,
+ subset: str | None = None,
+) -> list[str]:
+ """The globs the card declares for this split, under the named config."""
+ config = _pick_config(configs, subset)
+ if config is None:
+ return []
+ declared = config.get("data_files")
+ named = _patterns_for_split(declared, split) or _patterns_for_split(
+ declared, split, exact = False
+ )
+ folder = _config_folder(config)
+ # `_resolve_data_files` joins `data_dir` to the declared pattern whatever
+ # the two say, even where the pattern repeats the folder (`builder.py`).
+ return [_normalized_glob(f"{folder}/{p}" if folder else p) for p in named]
+
+
+def _normalized_glob(pattern: str) -> str:
+ """The pattern as the listing spells it, the way `normpath` would.
+
+ `list_repo_files` returns data/train.parquet, so ./data and staging/../data
+ have to lose their dots here or match nothing.
+ """
+ parts: list[str] = []
+ for part in pattern.strip().split("/"):
+ if part in ("", "."):
+ continue
+ if part == ".." and parts and parts[-1] != "..":
+ parts.pop()
+ continue
+ parts.append(part)
+ return "/".join(parts)
+
+
+def _config_folder(config: dict[str, Any]) -> str:
+ folder = config.get("data_dir")
+ return _normalized_glob(folder) if isinstance(folder, str) else ""
+
+
+def _config_scope(configs: list[dict[str, Any]], subset: str | None) -> str:
+ """The folder a config is confined to, for a card that names one and lists no files."""
+ config = _pick_config(configs, subset)
+ return _config_folder(config) if config else ""
+
+
+def _pick_config(configs: list[dict[str, Any]], subset: str | None) -> dict[str, Any] | None:
+ """The config a bare request lands on, the way the Hub resolves it.
+
+ Without a subset the answer is not always the one literally called `default`:
+ a card may flag another config as the default, and a card with a single
+ config uses it whatever its name (imdb ships only `plain_text`).
+ """
+ if subset:
+ # A card may carry both Foo and foo, so the exact one wins and a folded
+ # match is only a fallback. No config_name means the default one.
+ names = [(c, str(c.get("config_name") or DEFAULT_CONFIG)) for c in configs]
+ return next(
+ (c for c, name in names if name == subset),
+ next((c for c, name in names if name.lower() == subset.lower()), None),
+ )
+ flagged = next((c for c in configs if c.get("default") is True), None)
+ if flagged is not None:
+ return flagged
+ if len(configs) == 1:
+ return configs[0]
+ # Exactly `default`: Default and DEFAULT are configs to be asked for.
+ return next(
+ (c for c in configs if str(c.get("config_name") or DEFAULT_CONFIG) == DEFAULT_CONFIG),
+ None,
+ )
+
+
+def _glob_to_regex(pattern: str) -> re.Pattern[str]:
+ """A card glob as a regex: `*` stops at a folder boundary, `**` crosses it.
+
+ Matching on the literal prefix instead would let `data/train-*.parquet` claim
+ `data/train-notes.json`, which the recipe path then never reads.
+ """
+ parts: list[str] = []
+ i = 0
+ while i < len(pattern):
+ char = pattern[i]
+ if char == "*" and pattern[i + 1 : i + 2] == "*":
+ if pattern[i + 2 : i + 3] == "/":
+ # Whole folders, or none: `data/**/train.*` is not `data/nottrain.x`.
+ parts.append("(?:[^/]+/)*")
+ i += 3
+ else:
+ parts.append(".*")
+ i += 2
+ continue
+ if char == "*":
+ parts.append("[^/]*")
+ elif char == "?":
+ parts.append("[^/]")
+ elif char == "[":
+ end = _class_end(pattern, i)
+ if end < 0:
+ parts.append(r"\[")
+ else:
+ body = pattern[i + 1 : end].replace("\\", "\\\\")
+ # Only `!` negates a glob class; a leading `^` is a character.
+ if body[:1] == "!":
+ body = f"^{body[1:]}"
+ elif body[:1] == "^":
+ body = f"\\^{body[1:]}"
+ parts.append(f"[{body}]")
+ i = end
+ else:
+ parts.append(re.escape(char))
+ i += 1
+ return re.compile("".join(parts) + r"\Z")
+
+
+def _class_end(pattern: str, start: int) -> int:
+ """Index of the `]` closing the class opened at `start`, or -1 if unclosed."""
+ i = start + 1
+ if pattern[i : i + 1] in ("!", "^"):
+ i += 1
+ if pattern[i : i + 1] == "]":
+ i += 1
+ closing = pattern.find("]", i)
+ return closing if closing > start else -1
+
+
+def _ignored_by_the_loader(path: str, pattern: str) -> bool:
+ """Whether the loader drops this match by name, as dataset_info.json.
+
+ Those basenames leave a glob's matches unless the pattern is that file
+ itself (`data_files.FILES_TO_IGNORE`).
+ """
+ name = PurePosixPath(path).name
+ return name in _IGNORED_DATA_FILENAMES and PurePosixPath(pattern).name != name
+
+
+def _hidden_to_the_loader(path: str, pattern: str) -> bool:
+ """Whether a match sits in a part the loader skips unless it was asked for.
+
+ A hidden or dunder component the pattern does not carry itself drops the
+ match, so `**/*` never reaches .backup/ or __pycache__/ (`data_files`).
+ """
+ # A dot counts anywhere, a dunder only in a parent, so data/__train.json
+ # is a file the loader reads.
+ for prefix, parts in ((".", "parts"), ("__", "parent_parts")):
+ in_path = _marked_parts(path, prefix, parts)
+ in_pattern = _marked_parts(pattern, prefix, parts)
+ if len(in_path) != len(in_pattern):
+ return True
+ return False
+
+
+def _marked_parts(path: str, prefix: str, which: str) -> list[str]:
+ parsed = PurePosixPath(path)
+ parts = parsed.parts if which == "parts" else parsed.parent.parts
+ return [part for part in parts if part.startswith(prefix) and set(part) != {"."}]
+
+
+def _files_under_patterns(
+ patterns: list[str],
+ data_files: list[str],
+ *,
+ as_the_loader_would: bool = False,
+) -> list[str]:
+ """The files a glob takes, under the loader's rules for a card's own glob.
+
+ A pattern this module writes goes to a reader that skips nothing, so it is
+ judged on what it plainly matches.
+ """
+ matchers = [(pattern, _glob_to_regex(pattern)) for pattern in patterns]
+ return [
+ f
+ for f in data_files
+ if any(
+ m.match(f)
+ and not (
+ as_the_loader_would
+ and (_hidden_to_the_loader(f, p) or _ignored_by_the_loader(f, p))
+ )
+ for p, m in matchers
+ )
+ ]
+
+
+def _common_parent(paths: list[str]) -> str:
+ parts = [Path(path).parent.as_posix().split("/") for path in paths]
+ shared: list[str] = []
+ for segments in zip(*parts):
+ if len(set(segments)) != 1 or segments[0] == ".":
+ break
+ shared.append(segments[0])
+ return "/".join(shared)
+
+
+def _label_in_name(name: str, label: str) -> bool:
+ """The label standing on its own in a file name, separators and all.
+
+ Tokenizing the name instead would lose a label that carries a separator, so
+ validation_matched or sample-10BT could never match anything. The separators
+ are the loader's own (`local_options._SEP`), digits included, so train1 is
+ the train split while training is not.
+ """
+ return re.search(rf"(?:^|{_SEP}){re.escape(label)}(?:$|{_SEP})", name) is not None
+
+
+def _split_folder(lowered_path: str, labels: tuple[str, ...]) -> bool:
+ """A folder standing for this split, qualified or not: train, train_a, en-train.
+
+ The loader reads a separator-qualified folder as the split too
+ (`local_options._DIR_NAME_KEYWORD_PATTERNS`), so an exact component match
+ would leave train_a ranking as nothing.
+ """
+ return any(
+ _label_in_name(part, label) for part in lowered_path.split("/")[:-1] for label in labels
+ )
+
+
+def _split_labels(split_lower: str) -> tuple[str, ...]:
+ """The names `datasets` accepts for this split: dev is validation, training is train."""
+ for canonical, aliases in _SPLIT_KEYWORDS.items():
+ if split_lower == canonical or split_lower in aliases:
+ return aliases
+ return (split_lower,)
+
+
+def _carries_another_split(path: str, split_lower: str) -> bool:
+ """The path is labelled with a split, and it is not the one being asked for."""
+ wanted = set(_split_labels(split_lower))
+ return any(
+ canonical not in wanted and _split_rank(path, canonical) <= 1
+ for canonical in _SPLIT_KEYWORDS
+ )
+
+
+def _split_rank(path: str, split_lower: str) -> int:
+ """0 the split is a folder, 1 the file name carries it, 2 neither.
+
+ The name is read as whole labels rather than by where the split sits in it,
+ so questions_train_000.jsonl counts as train, and `datasets` own aliases
+ count too, so dev.jsonl is validation.
+ """
+ lowered = path.lower()
+ labels = _split_labels(split_lower)
+ if _split_folder(lowered, labels):
+ return 0
+ # Only the final extension comes off: questions.train.parquet keeps its split.
+ stem = Path(lowered).stem
+ if any(_label_in_name(stem, label) for label in labels):
+ return 1
+ return 2
+
+
+def _in_subset(data_files: list[str], subset: str | None, split_lower: str) -> list[str]:
+ """The files carrying this config label, as a folder or in the file name.
+
+ A repo may encode its configs in the names instead of the layout, as
+ main-train.parquet beside socratic-train.parquet. A folder carrying the
+ label is the config whatever its files are called; a name only carrying it
+ while belonging to no part of the split is a coincidence, not the config,
+ so it is dropped rather than allowed to decide the answer.
+ """
+ if not subset:
+ return data_files
+ subset_lower = subset.lower()
+ # Exactly first: a repo may hold Foo/ beside foo/, and folding the two
+ # together would read both configs as one.
+ in_folder = [f for f in data_files if subset in f.split("/")[:-1]] or [
+ f for f in data_files if subset_lower in f.lower().split("/")[:-1]
+ ]
+ if in_folder:
+ return in_folder
+ named = [f for f in data_files if _label_in_name(Path(f.lower()).stem, subset_lower)]
+ if any(_split_rank(f, split_lower) <= 1 for f in named):
+ return named
+ if any(_split_rank(f, split_lower) <= 1 for f in data_files):
+ return data_files
+ return named or data_files
+
+
+def _select_best_file(
+ data_files: list[str],
+ split: str = DEFAULT_SPLIT,
+ subset: str | None = None,
+) -> str | None:
if not data_files:
return None
split_lower = split.lower()
+ return sorted(
+ _in_subset(data_files, subset, split_lower),
+ key = lambda p: (_split_rank(p, split_lower), len(p)),
+ )[0]
+
+
+def _pattern_fits_the_split(
+ data_files: list[str], wanted: set[str], root: str, pattern: str
+) -> bool:
+ """The pattern must take every wanted file under the root, and no other file there.
+
+ A folder may name one split several ways at once (train-part.parquet beside
+ questions_train.parquet), so a glob built from whichever file was picked can
+ drop the rest; a loose glob can just as easily swallow a neighbouring split.
+ Read relative to the root the pattern is anchored at, so a split spread over
+ sibling folders is only accepted by a pattern that reaches all of them.
+ Judged against the WHOLE listing, since a candidate checked only against the
+ subset slice can still match another subset's files.
+ """
+ matcher = _glob_to_regex(pattern)
+ prefix = f"{root}/" if root and root != "." else ""
+ for path in data_files:
+ if not path.startswith(prefix):
+ continue
+ if bool(matcher.match(path[len(prefix) :])) != (path in wanted):
+ return False
+ return True
+
- def score(path: str) -> tuple[int, int]:
- name = path.lower()
- if f"/{split_lower}/" in name:
- return (0, len(path))
- if (
- f"_{split_lower}." in name
- or f"-{split_lower}." in name
- or f"/{split_lower}." in name
- or f"/{split_lower}_" in name
- or f"/{split_lower}-" in name
- ):
- return (1, len(path))
- return (2, len(path))
+def _candidate_patterns(stem: str, suffix: str, split_lower: str) -> list[str]:
+ """Globs for this split's files in one folder, narrowest first.
+
+ Built from whichever of the split's names this file actually uses, since a
+ validation shard may well be called dev.
+ """
+ stem_lower = stem.lower()
+ candidates: list[str] = []
+ for label in _split_labels(split_lower):
+ if stem_lower.startswith((f"{label}-", f"{label}_", f"{label}.")):
+ candidates.append(f"{stem[: len(label) + 1]}*{suffix}")
+ if stem_lower == label or stem_lower.endswith((f"_{label}", f"-{label}")):
+ # Keep the star: train.jsonl may be sharded beside train_2.jsonl.
+ candidates.append(f"{stem}*{suffix}")
+ at = stem_lower.find(label)
+ if at >= 0:
+ candidates.append(f"*{stem[at : at + len(label)]}*{suffix}")
+ # Last and narrowest: the whole name, which keeps train-main off
+ # train-socratic.
+ candidates.append(f"{stem}*{suffix}")
+ return list(dict.fromkeys(candidates))
+
+
+def _with_data_extension(pattern: str, suffix: str) -> str:
+ """A card glob ends where the file name does; the reader picks by extension."""
+ current = Path(pattern).suffix
+ if current.lower() in DATA_EXTS:
+ return pattern
+ if current and any(char in current for char in "*?["):
+ pattern = pattern[: -len(current)]
+ # fsspec refuses data/**.parquet: `**` has to be a whole component.
+ if pattern.endswith("**"):
+ pattern = f"{pattern}/*"
+ return f"{pattern}{suffix}"
+
+
+def _ext_rank(suffix: str) -> int:
+ """Where `datasets` puts this extension in a draw: parquet, jsonl, json, csv."""
+ return -DATA_EXTS.index(suffix) if suffix in DATA_EXTS else -99
+
+
+def _dominant_suffix(paths: list[str]) -> str:
+ """The format the loader would build this split from: most files, parquet on ties.
+
+ One broad card glob can cover several formats, and picking whichever name
+ sorts first would point the recipe at a format the split does not use
+ (`local_options._one_module`).
+ """
+ counts: dict[str, int] = {}
+ # The window first, exactly as the loader slices it, and metadata dropped
+ # only within it: a shard past the window is one the loader never sees.
+ window = sorted(paths)[:_MAX_MODULE_INFERENCE_FILES]
+ voting = [p for p in window if Path(p).name.lower() not in _METADATA_FILENAMES] or window
+ for path in voting:
+ # A vote for every suffix in the name and one extension per vote, as
+ # `load.infer_module_for_data_files_list` counts them: a.csv.parquet is
+ # a csv as much as a parquet, and .json lends .jsonl none of its votes.
+ for part in Path(path).name.lower().split(".")[1:]:
+ counts[f".{part}"] = counts.get(f".{part}", 0) + 1
+ if not counts:
+ return ""
+
+ # The builder widens only once its extension has won on its own. A winner no
+ # file actually ends in cannot be written as a glob, so the next one that can
+ # is taken rather than a pattern that matches nothing.
+ for winner in sorted(counts, key = lambda s: (counts[s], _ext_rank(s)), reverse = True):
+ group = _builder_exts(winner)
+ named = next((Path(p).suffix for p in paths if Path(p).suffix.lower() in group), "")
+ if named:
+ return named
+ return ""
+
+
+# Extensions one builder reads: `datasets` loads .json and .jsonl with the same one.
+_BUILDER_EXTS = ((".json", ".jsonl"),)
+
+
+def _builder_exts(suffix: str) -> tuple[str, ...]:
+ lowered = suffix.lower()
+ for group in _BUILDER_EXTS:
+ if lowered in group:
+ return group
+ return (lowered,)
+
+
+def _of_suffix(paths: list[str], suffix: str) -> list[str]:
+ group = _builder_exts(suffix)
+ return [p for p in paths if Path(p).suffix.lower() in group]
+
+
+def _extension_glob(paths: list[str], suffix: str, repo_files: list[str]) -> str:
+ """What the pattern should end in to keep every file this builder reads.
+
+ A split over both .json and .jsonl needs the shared start of the two, which
+ leaves the glob open at the end; where the repo holds a a.json.gz that glob
+ would read it too, so there the winning extension is used alone.
+ """
+ group = set(_builder_exts(suffix))
+ used = {Path(path).suffix.lower() for path in paths} & group
+ if len(used) < 2:
+ return suffix
+ stem = os.path.commonprefix(sorted(used))
+ matcher = re.compile(rf"{re.escape(stem)}[^/]*\Z", re.IGNORECASE)
+ # Only what the emitted pattern can reach: it is anchored at the folder the
+ # declared files share, so a name collision anywhere else is not its problem.
+ scope = _common_parent(paths)
+ prefix = f"{scope}/" if scope else ""
+ if any(
+ path.startswith(prefix)
+ and Path(path).suffix.lower() not in group
+ and matcher.search(Path(path).name)
+ for path in repo_files
+ ):
+ return suffix
+ return f"{stem}*"
+
+
+def _common_name_prefix(paths: list[str]) -> str:
+ """What the file names share before the extension, which the glob supplies."""
+ return os.path.commonprefix([Path(path).stem for path in paths]) if paths else ""
+
+
+def _name_initials(paths: list[str]) -> str:
+ """The first characters of these file names, as a glob class body.
+
+ Only letters and digits, so nothing in the class can be read as syntax.
+ """
+ initials = sorted({Path(path).name[:1] for path in paths if Path(path).name[:1].isalnum()})
+ if not initials or len(initials) != len({Path(path).name[:1] for path in paths}):
+ return ""
+ return "".join(initials)
+
+
+def _staged_split_files(
+ data_files: list[str],
+ labels: tuple[str, ...],
+ scope: str = "",
+) -> set[str]:
+ """This split's shards, when the repo is laid out the way the loader looks first.
+
+ Split inference is staged and the sharded names come first: once
+ data/train-00000-of-00001.parquet is there, a train-named file elsewhere is
+ not in the split (`local_options._grouped_splits`).
+ """
+ # The loader reads a config's files relative to its `data_dir`, and the
+ # sharded names it looks for start at `data/`, so the scope comes off first.
+ prefix = f"{scope}/" if scope else ""
+ inside = [path[len(prefix) :] for path in data_files if path.startswith(prefix)]
+ grouped = _sharded_splits([PurePosixPath(path) for path in inside])
+ if not grouped:
+ return set()
+ staged = {
+ f"{prefix}{path.as_posix()}"
+ for split, paths in grouped.items()
+ if split.lower() in labels
+ for path in paths
+ }
+ return staged if staged < set(data_files) else set()
- return sorted(data_files, key = score)[0]
+
+def _anchor(dataset_name: str, root: str) -> str:
+ """The recipe path up to the folder a pattern is written relative to."""
+ base = f"datasets/{dataset_name}"
+ return f"{base}/{root}" if root and root != "." else base
+
+
+def _name_finals(paths: list[str]) -> str:
+ """The last characters of these names before the extension, as a class body."""
+ finals = {Path(path).stem[-1:] for path in paths}
+ return "".join(sorted(finals)) if finals and all(c.isalnum() for c in finals) else ""
+
+
+def _widened_declared_pattern(
+ declared_files: list[str],
+ data_files: list[str],
+ split_lower: str,
+ suffix: str,
+ *,
+ fallback_glob: bool = False,
+) -> str:
+ """One glob covering several declared ones, keeping the split out of it if it can.
+
+ The folder holding them all may hold the other splits too, which is what the
+ split-named form avoids; it is only used when it takes the declared files
+ and nothing else.
+ """
+ parent = _common_parent(declared_files)
+ base = f"{parent}/**" if parent else "**"
+ prefix = f"{parent}/" if parent else ""
+ wanted = set(declared_files)
+ # The split may be written into the file names or into the folders between
+ # here and them, so try both before giving up on keeping it.
+ # Whichever of the split's names the files use: a validation set may well be
+ # declared as dev-*.parquet.
+ candidates = [f"{base}/*{label}*{suffix}" for label in _split_labels(split_lower)]
+ candidates += [f"{prefix}**/*{label}*/**/*{suffix}" for label in _split_labels(split_lower)]
+ # Declared files that say nothing about the split may still share a name the
+ # neighbouring splits do not, as data/part-a beside data/part-b, or
+ # sets/a/part beside sets/b/part with a test file in sets/c.
+ shared_name = _common_name_prefix(declared_files)
+ if shared_name:
+ candidates.append(f"{prefix}{shared_name}*{suffix}")
+ candidates.append(f"{prefix}**/{shared_name}*{suffix}")
+ # Unrelated names still start somewhere the neighbours do not. A class is the
+ # only union the reader understands; it rejects `{a,b}` outright.
+ initials = _name_initials(declared_files)
+ finals = _name_finals(declared_files)
+ if initials:
+ candidates.append(f"{prefix}**/[{initials}]*{suffix}")
+ # Names that start where a neighbour also starts may still end elsewhere:
+ # apple and banana beside avocado share their first letter with it but not
+ # their last, and one class at each end separates them.
+ if finals:
+ candidates.append(f"{prefix}**/*[{finals}]{suffix}")
+ if initials and finals:
+ candidates.append(f"{prefix}**/[{initials}]*[{finals}]{suffix}")
+ # A hand-written mapping may name files no glob can gather. Each one still
+ # names itself, so there is always something to fall back to that reads only
+ # what the card declared.
+ candidates += sorted(wanted)
+ for candidate in candidates:
+ if set(_files_under_patterns([candidate], data_files)) == wanted:
+ return candidate
+ # Nothing names these files and only these files: train aa and bb beside
+ # test ab and ba defeat every class a glob can carry. Against a card, that
+ # is refused rather than read wide or short; against a guess from the file
+ # names, the folder is still covered.
+ return f"{base}/*{suffix}" if fallback_glob else ""
def _resolve_seed_hf_path(
dataset_name: str,
data_files: list[str],
split: str = DEFAULT_SPLIT,
+ subset: str | None = None,
+ configs: list[dict[str, Any]] | None = None,
+ repo_files: list[str] | None = None,
) -> str | None:
- selected = _select_best_file(data_files, split)
+ declared = _declared_split_patterns(configs or [], split, subset)
+ declared_files = _files_under_patterns(declared, data_files, as_the_loader_would = True)
+ # The card's own mapping beats any guess from the folder names, as long as it
+ # resolves to files that are really there.
+ if declared_files:
+ suffix = _dominant_suffix(declared_files)
+ declared_files = _of_suffix(declared_files, suffix)
+ exact = suffix
+ suffix = _extension_glob(declared_files, suffix, repo_files or data_files)
+ if suffix == exact and len({Path(f).suffix.lower() for f in declared_files}) > 1:
+ # The card put both of the builder's extensions in this split and no
+ # glob can take the two without also taking the file that made the
+ # combined form unsafe. Refused, rather than read short.
+ return None
+ pattern = ""
+ if len(declared) == 1:
+ pattern = _with_data_extension(declared[0], suffix)
+ # A split spread over several declared globs cannot be written as one, and
+ # neither can a rewritten pattern that takes anything other than what the
+ # card declared: the reader skips no hidden folder the loader would, so
+ # the rewrite has to exclude one itself.
+ if not pattern or set(declared_files) != set(_files_under_patterns([pattern], data_files)):
+ pattern = _widened_declared_pattern(declared_files, data_files, split.lower(), suffix)
+ # An unrepresentable mapping is refused, not approximated: the endpoint
+ # answers 422 and the recipe is never pointed at the wrong rows.
+ return f"datasets/{dataset_name}/{pattern}" if pattern else None
+
+ # Without a card the subset is only a label, so narrowing first keeps the
+ # pattern from widening back over another config.
+ folder = _config_scope(configs or [], subset)
+ scoped = _in_subset(
+ [f for f in data_files if f.startswith(f"{folder}/")] or data_files
+ if folder
+ else data_files,
+ subset,
+ split.lower(),
+ )
+ split_lower = split.lower()
+ labels = _split_labels(split_lower)
+ # Sharded names come first and stop the search, whatever else is labelled.
+ staged = _staged_split_files(scoped, labels, folder)
+ selected = _select_best_file(sorted(staged) or scoped, split)
if not selected:
return None
- ext = Path(selected).suffix.lower()
+ suffix = Path(selected).suffix
+ ext = suffix.lower()
if ext not in DATA_EXTS:
return f"datasets/{dataset_name}/{selected}"
parent = Path(selected).parent.as_posix()
- if not parent or parent == ".":
- return f"datasets/{dataset_name}/**/*{ext}"
- return f"datasets/{dataset_name}/{parent}/**/*{ext}"
+ base = f"datasets/{dataset_name}"
+ if parent and parent != ".":
+ base = f"{base}/{parent}"
+
+ if _split_folder(selected.lower(), labels):
+ # train_a and train_b are one split, so rooting at either drops half.
+ folders = staged or {f for f in scoped if _split_rank(f, split_lower) == 0}
+ root = _common_parent(sorted(folders | {selected}))
+ if root != parent:
+ for label in labels:
+ # The four shapes `local_options._DIR_NAME_KEYWORD_PATTERNS`
+ # accepts, the last for a label between separators, a_train_x.
+ for shape in (
+ f"**/{label}/**/*{ext}",
+ f"**/{label}{_SEP}*/**/*{ext}",
+ f"**/*{_SEP}{label}/**/*{ext}",
+ f"**/*{_SEP}{label}{_SEP}*/**/*{ext}",
+ # train/ beside sets/train_a/ needs a looser form, still
+ # only taken where the listing bears it out.
+ f"**/{label}*/**/*{ext}",
+ f"**/*{label}*/**/*{ext}",
+ ):
+ if _pattern_fits_the_split(data_files, folders, root, shape):
+ return f"{_anchor(dataset_name, root)}/{shape}"
+ else:
+ # The files this request is for: in the chosen subset, and of this split.
+ wanted = staged or {f for f in scoped if _split_rank(f, split_lower) <= 1}
+ # A split sharded over sibling folders is anchored at the folder they
+ # share: without a card the loader reads those as one split, and a
+ # requested subset has already narrowed `scoped`.
+ spread = any(Path(f).parent.as_posix() != parent for f in wanted)
+ root = _common_parent(sorted(wanted | {selected})) if spread else parent
+ stem = Path(selected).name[: -len(suffix)]
+ for candidate in _candidate_patterns(stem, suffix, split_lower):
+ pattern = f"**/{candidate}" if spread else candidate
+ if _pattern_fits_the_split(data_files, wanted, root, pattern):
+ return f"{_anchor(dataset_name, root)}/{pattern}"
+ # dev-0 beside validation-0 is one split, and no pattern built from the
+ # chosen file gathers the other, so the declared-mapping widening is
+ # reused. Only where every neighbour is labelled: an unlabelled file may
+ # belong to this split, and only the broad glob would keep it.
+ prefix = f"{root}/" if root and root != "." else ""
+ neighbours = [f for f in data_files if f.startswith(prefix) and f not in wanted]
+ if len(wanted) > 1 and all(_carries_another_split(f, split_lower) for f in neighbours):
+ widened = _widened_declared_pattern(
+ sorted(wanted), data_files, split_lower, suffix, fallback_glob = True
+ )
+ return f"datasets/{dataset_name}/{widened}"
+ return f"{base}/**/*{ext}"
def _build_stream_load_kwargs(
@@ -356,9 +1055,25 @@ def inspect_seed_dataset(
) from exc
preview_rows: list[dict[str, Any]] = []
- data_files = _list_hf_data_files(dataset_name = dataset_name, token = token)
-
- selected_file = _select_best_file(data_files, split)
+ repo_files = _list_hf_repo_files(dataset_name = dataset_name, token = token)
+ data_files = [file for file in repo_files if file.lower().endswith(DATA_EXTS)]
+ configs = _list_hf_dataset_configs(dataset_name = dataset_name, token = token)
+
+ # Preview the same files the recipe will read, so the rows on screen are not
+ # from a config the resolved path excludes.
+ declared_files = _files_under_patterns(
+ _declared_split_patterns(configs, split, subset),
+ data_files,
+ as_the_loader_would = True,
+ )
+ # Same format the resolved path will read, or the rows on screen come from a
+ # file the recipe never opens.
+ declared_files = _of_suffix(declared_files, _dominant_suffix(declared_files))
+ # A config that only names a folder declares no files, and its name need not
+ # be that folder, so the fallback previews the folder rather than the repo.
+ folder = _config_scope(configs, subset)
+ in_folder = [f for f in data_files if f.startswith(f"{folder}/")] if folder else []
+ selected_file = _select_best_file(declared_files or in_folder or data_files, split, subset)
if selected_file:
try:
single_file_kwargs = _build_stream_load_kwargs(
@@ -406,7 +1121,9 @@ def inspect_seed_dataset(
if not data_files:
resolved_path = f"datasets/{dataset_name}/**/*.parquet"
else:
- resolved_path = _resolve_seed_hf_path(dataset_name, data_files, split)
+ resolved_path = _resolve_seed_hf_path(
+ dataset_name, data_files, split, subset, configs, repo_files
+ )
if not resolved_path:
raise HTTPException(status_code = 422, detail = "unable to resolve seed dataset path")
diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.py
index 6a4d32b470c..94680698505 100644
--- a/studio/backend/tests/test_data_recipe_seed.py
+++ b/studio/backend/tests/test_data_recipe_seed.py
@@ -432,6 +432,1128 @@ def test_a_backend_executed_seed_resolves_the_endpoint_on_the_backend(monkeypatc
_resolve_seed_endpoint({"seed_config": "nope"})
+_GSM8K_FILES = [
+ "main/test-00000-of-00001.parquet",
+ "main/train-00000-of-00001.parquet",
+ "socratic/test-00000-of-00001.parquet",
+ "socratic/train-00000-of-00001.parquet",
+]
+
+
+ ("files", "split", "subset", "expected"),
+ [
+ # No card: the loader reads every train-named file as one split.
+ (_GSM8K_FILES, "train", None, "datasets/org/repo/**/train-*.parquet"),
+ (_GSM8K_FILES, "test", "main", "datasets/org/repo/main/test-*.parquet"),
+ (_GSM8K_FILES, "train", "socratic", "datasets/org/repo/socratic/train-*.parquet"),
+ (_GSM8K_FILES, "train", "default", "datasets/org/repo/**/train-*.parquet"),
+ (
+ ["data/test-00000-of-00001.parquet", "data/train-00000-of-00002.parquet"],
+ "train",
+ "default",
+ "datasets/org/repo/data/train-*.parquet",
+ ),
+ (["test.csv", "train.csv"], "train", None, "datasets/org/repo/train*.csv"),
+ (["Train_0.jsonl", "Test_0.jsonl"], "train", None, "datasets/org/repo/Train_*.jsonl"),
+ (
+ ["raw/gsm_test.jsonl", "raw/gsm_train.jsonl"],
+ "train",
+ None,
+ "datasets/org/repo/raw/gsm_train*.jsonl",
+ ),
+ # A split named by the file itself can still be sharded, so the pattern has
+ # to reach the siblings instead of pinning the first shard.
+ (
+ ["train.jsonl", "train_2.jsonl", "test.jsonl"],
+ "train",
+ None,
+ "datasets/org/repo/train*.jsonl",
+ ),
+ (
+ ["raw/gsm_test.jsonl", "raw/gsm_train.jsonl", "raw/gsm_train_2.jsonl"],
+ "train",
+ None,
+ "datasets/org/repo/raw/gsm_train*.jsonl",
+ ),
+ (
+ ["en/test/0000.parquet", "en/train/0000.parquet", "fr/train/0000.parquet"],
+ "train",
+ "fr",
+ "datasets/org/repo/fr/train/**/*.parquet",
+ ),
+ (["data/part-0.parquet"], "train", "main", "datasets/org/repo/data/**/*.parquet"),
+ ],
+)
+def test_seed_hf_path_keeps_the_split_and_subset(
+ monkeypatch, tmp_path, files, split, subset, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ assert seed_route._resolve_seed_hf_path("org/repo", files, split, subset) == expected
+
+
+def test_seed_preview_file_comes_from_the_chosen_subset(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ assert (
+ seed_route._select_best_file(_GSM8K_FILES, "train", "socratic")
+ == "socratic/train-00000-of-00001.parquet"
+ )
+
+
+# fineweb-edu: the config name is not a folder, so a folder-name guess reads a
+# different config entirely.
+_SAMPLE_FILES = [
+ "data/CC-MAIN-2013-20/train-00000-of-00014.parquet",
+ "sample/10BT/000_00000.parquet",
+ "sample/10BT/001_00000.parquet",
+]
+_SAMPLE_CONFIGS = [
+ {"config_name": "default", "data_files": [{"split": "train", "path": "data/*/*"}]},
+ {"config_name": "sample-10BT", "data_files": [{"split": "train", "path": "sample/10BT/*"}]},
+]
+
+
+ ("subset", "expected"),
+ [
+ ("sample-10BT", "datasets/org/repo/sample/10BT/*.parquet"),
+ (None, "datasets/org/repo/data/*/*.parquet"),
+ ],
+)
+def test_seed_hf_path_follows_the_dataset_card_config_mapping(
+ monkeypatch, tmp_path, subset, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ resolved = seed_route._resolve_seed_hf_path(
+ "org/repo", _SAMPLE_FILES, "train", subset, _SAMPLE_CONFIGS
+ )
+ assert resolved == expected
+
+
+def test_seed_hf_path_ignores_a_card_mapping_pointing_at_nothing(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [{"config_name": "main", "data_files": [{"split": "train", "path": "gone/*"}]}]
+ resolved = seed_route._resolve_seed_hf_path("org/repo", _GSM8K_FILES, "train", "main", configs)
+ assert resolved == "datasets/org/repo/main/train-*.parquet"
+
+
+ "data_files",
+ [
+ "mapped/train-00000.parquet",
+ ["mapped/train-00000.parquet"],
+ {"train": "mapped/train-*.parquet"},
+ [{"split": "train", "path": "mapped/train-*.parquet"}],
+ ],
+)
+def test_seed_hf_path_reads_every_card_data_files_shape(monkeypatch, tmp_path, data_files):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["mapped/train-00000.parquet", "other/train-00000.parquet"]
+ configs = [{"config_name": "unrelated", "data_files": data_files}]
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", "unrelated", configs)
+ assert resolved.startswith("datasets/org/repo/mapped/")
+
+
+ ("configs", "expected"),
+ [
+ # A card with one config uses it whatever it is called.
+ (
+ [
+ {
+ "config_name": "plain_text",
+ "data_files": [{"split": "train", "path": "pt/train-*"}],
+ }
+ ],
+ "datasets/org/repo/pt/train-*.parquet",
+ ),
+ # Another config can be flagged as the default one.
+ (
+ [
+ {"config_name": "a", "data_files": [{"split": "train", "path": "a/train-*"}]},
+ {
+ "config_name": "b",
+ "default": True,
+ "data_files": [{"split": "train", "path": "pt/train-*"}],
+ },
+ ],
+ "datasets/org/repo/pt/train-*.parquet",
+ ),
+ ],
+)
+def test_seed_hf_path_finds_the_default_config_without_a_subset(
+ monkeypatch, tmp_path, configs, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["pt/train-0.parquet", "a/train-0.parquet"]
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) == expected
+
+
+def test_seed_card_globs_keep_their_character_classes(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/train-2-of-9.parquet", "data/train-7-of-9.parquet"]
+ assert seed_route._files_under_patterns(["data/train-[0-4]*.parquet"], files) == [
+ "data/train-2-of-9.parquet"
+ ]
+ assert seed_route._files_under_patterns(["data/[!x]*.parquet"], files) == files
+ # An unclosed bracket is a literal, not a syntax error.
+ assert seed_route._files_under_patterns(["a[b.parquet"], ["a[b.parquet"]) == ["a[b.parquet"]
+
+
+def test_seed_hf_path_covers_a_split_declared_across_two_folders(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["sets/a/train-0.parquet", "sets/b/train-0.parquet", "other/test-0.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": ["sets/a/train-*.parquet", "sets/b/train-*.parquet"]}
+ ],
+ }
+ ]
+ # One glob cannot name two folders, so cover the folder holding both rather
+ # than dropping one of them, keeping the split in the pattern.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/sets/**/*train*.parquet"
+ )
+
+
+_LABELLED_FILES = [
+ "main-train.parquet",
+ "main-test.parquet",
+ "socratic-train.parquet",
+ "socratic-test.parquet",
+]
+
+
+ ("subset", "expected"),
+ [
+ ("socratic", "datasets/org/repo/socratic-train*.parquet"),
+ ("main", "datasets/org/repo/main-train*.parquet"),
+ ],
+)
+def test_seed_hf_path_reads_a_subset_labelled_in_the_file_name(
+ monkeypatch, tmp_path, subset, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ resolved = seed_route._resolve_seed_hf_path("org/repo", _LABELLED_FILES, "train", subset)
+ assert resolved == expected
+
+
+ ("files", "split", "expected"),
+ [
+ (["data/dev.jsonl", "data/train.jsonl"], "validation", "datasets/org/repo/data/dev*.jsonl"),
+ (
+ ["data/valid-0.parquet", "data/train-0.parquet"],
+ "validation",
+ "datasets/org/repo/data/valid-*.parquet",
+ ),
+ (
+ ["data/training-0.parquet", "data/test-0.parquet"],
+ "train",
+ "datasets/org/repo/data/*train*.parquet",
+ ),
+ ],
+)
+def test_seed_hf_path_follows_the_split_aliases_datasets_uses(
+ monkeypatch, tmp_path, files, split, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ assert seed_route._resolve_seed_hf_path("org/repo", files, split) == expected
+
+
+ ("files", "expected"),
+ [
+ # A folder qualified by a separator is the split folder too.
+ (["train_a/0.parquet", "test/0.parquet"], "datasets/org/repo/train_a/**/*.parquet"),
+ (
+ ["data/train/0.parquet", "data/test/0.parquet"],
+ "datasets/org/repo/data/train/**/*.parquet",
+ ),
+ # "pretrain" is a different word, so train/ still wins.
+ (["pretrain/0.parquet", "train/0.parquet"], "datasets/org/repo/train/**/*.parquet"),
+ ],
+)
+def test_seed_hf_path_reads_a_qualified_split_folder(monkeypatch, tmp_path, files, expected):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train") == expected
+
+
+def test_seed_hf_path_widens_through_an_alias_named_split(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["sets/a/dev-0.parquet", "sets/b/dev-0.parquet", "sets/a/train-0.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "validation", "path": ["sets/a/dev-*.parquet", "sets/b/dev-*.parquet"]},
+ {"split": "train", "path": "sets/a/train-*.parquet"},
+ ],
+ }
+ ]
+ # The files say dev, not validation, so the widened form has to say dev too
+ # or it takes the train shard with it.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "validation", None, configs)
+ == "datasets/org/repo/sets/**/*dev*.parquet"
+ )
+
+
+def test_seed_hf_path_counts_a_digit_as_a_label_separator(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ # The loader's own separators include digits, so train1 is train.
+ files = ["data/train-0.parquet", "data/train1.parquet", "data/test-0.parquet"]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train")
+ == "datasets/org/repo/data/*train*.parquet"
+ )
+ assert (
+ seed_route._resolve_seed_hf_path(
+ "org/repo", ["data/train-0.parquet", "data/testing-0.parquet"], "train"
+ )
+ == "datasets/org/repo/data/train-*.parquet"
+ )
+
+
+def test_seed_hf_path_keeps_a_split_spread_over_sibling_folders(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["sets/a/part.parquet", "sets/b/part.parquet", "sets/c/test.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": ["sets/a/part.parquet", "sets/b/part.parquet"]},
+ {"split": "test", "path": "sets/c/test.parquet"},
+ ],
+ }
+ ]
+ # Nothing names the split, but the two declared files share a name the test
+ # file does not, so the union stays off sets/c.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/sets/**/part*.parquet"
+ )
+
+
+def test_seed_hf_path_unions_declared_files_with_unrelated_names(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["sets/a/part-a.parquet", "sets/b/chunk-b.parquet", "sets/c/test.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": ["sets/a/part-a.parquet", "sets/b/chunk-b.parquet"]},
+ {"split": "test", "path": "sets/c/test.parquet"},
+ ],
+ }
+ ]
+ # Nothing is shared but the first letters, and a class is the only union the
+ # reader understands: it rejects {a,b} outright.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/sets/**/[cp]*.parquet"
+ )
+
+
+def test_seed_format_inference_stops_where_the_loader_stops(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ # 200 csv sort ahead of 201 parquet, and the loader only looks at the first
+ # 200, so it builds csv and the recipe has to agree.
+ files = [f"data/{i:03d}.csv" for i in range(200)] + [
+ f"data/z{i:03d}.parquet" for i in range(201)
+ ]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/data/*.csv"
+ )
+
+
+ ("files", "expected"),
+ [
+ # Most files wins, so the long csv name does not decide the format.
+ (
+ ["data/a-very-long-name.csv", "data/z.parquet", "data/y.parquet"],
+ "datasets/org/repo/data/*.parquet",
+ ),
+ (["data/a.csv", "data/z.parquet"], "datasets/org/repo/data/*.parquet"),
+ (["data/a.csv", "data/b.csv"], "datasets/org/repo/data/*.csv"),
+ ],
+)
+def test_seed_hf_path_picks_the_format_the_loader_would_build(
+ monkeypatch, tmp_path, files, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) == expected
+
+
+def test_seed_hf_path_combines_bare_and_explicit_train_entries(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ "data/part-a.parquet",
+ {"split": "train", "path": "data/part-b.parquet"},
+ ],
+ }
+ ]
+ assert seed_route._declared_split_patterns(configs, "train") == [
+ "data/part-a.parquet",
+ "data/part-b.parquet",
+ ]
+ files = ["data/part-a.parquet", "data/part-b.parquet", "data/test-0.parquet"]
+ # Neither declared file names the split, but they share a name the test file
+ # does not, so the union stays exact.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/data/part-*.parquet"
+ )
+
+
+def test_seed_hf_path_keeps_a_folder_subset_whose_files_are_generic(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ # en/ is the config even though its file says nothing about the split, so
+ # other/train.parquet must not win it.
+ files = ["en/data.parquet", "other/train.parquet"]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "en")
+ == "datasets/org/repo/en/**/*.parquet"
+ )
+
+
+ ("subset", "expected"),
+ [
+ ("main", "datasets/org/repo/train-main*.parquet"),
+ ("socratic", "datasets/org/repo/train-socratic*.parquet"),
+ ],
+)
+def test_seed_hf_path_keeps_a_subset_written_after_the_split(
+ monkeypatch, tmp_path, subset, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["train-main.parquet", "test-main.parquet", "train-socratic.parquet"]
+ # train-*.parquet fits the subset slice but takes the other subset with it,
+ # so the candidate has to be judged against the whole listing.
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset) == expected
+
+
+ ("subset", "expected"),
+ [
+ ("french", "datasets/org/repo/fr/train*.parquet"),
+ ("english", "datasets/org/repo/en/train*.parquet"),
+ ],
+)
+def test_seed_hf_path_scopes_a_config_to_its_data_dir(monkeypatch, tmp_path, subset, expected):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [
+ {"config_name": "english", "data_dir": "en"},
+ {"config_name": "french", "data_dir": "fr"},
+ ]
+ files = ["en/train.parquet", "en/test.parquet", "fr/train.parquet", "fr/test.parquet"]
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset, configs) == expected
+
+
+def test_seed_hf_path_reads_data_files_relative_to_data_dir(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [
+ {
+ "config_name": "a",
+ "data_dir": "d",
+ "data_files": [{"split": "train", "path": "train-*.parquet"}],
+ }
+ ]
+ files = ["d/train-0.parquet", "d/test-0.parquet"]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "a", configs)
+ == "datasets/org/repo/d/train-*.parquet"
+ )
+
+
+def test_seed_format_vote_ignores_folder_metadata(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+ files = ["data/metadata.csv", "data/metadata2.csv", "data/shard.parquet"]
+ # metadata.csv never decides a builder for the loader, so it cannot here.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/data/*.parquet"
+ )
+
+
+ ("subset", "expected"),
+ [
+ ("Foo", "datasets/org/repo/F/train.parquet"),
+ ("foo", "datasets/org/repo/f/train.parquet"),
+ ],
+)
+def test_seed_hf_path_matches_config_names_case_sensitively(
+ monkeypatch, tmp_path, subset, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [
+ {"config_name": "Foo", "data_files": [{"split": "train", "path": "F/train.parquet"}]},
+ {"config_name": "foo", "data_files": [{"split": "train", "path": "f/train.parquet"}]},
+ ]
+ files = ["F/train.parquet", "f/train.parquet"]
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset, configs) == expected
+
+
+def test_seed_hf_path_keeps_a_hand_listed_split_off_its_neighbours(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/apple.parquet", "data/banana.parquet", "data/avocado.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": ["data/apple.parquet", "data/banana.parquet"]},
+ {"split": "test", "path": "data/avocado.parquet"},
+ ],
+ }
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["data/apple.parquet", "data/banana.parquet"]
+
+
+def test_seed_hf_path_never_widens_a_declared_split_over_another(monkeypatch, tmp_path):
+ """No glob names these two alone, so the recipe reads what it can name."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/ax.parquet", "data/bx.parquet", "data/cx.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": ["data/ax.parquet", "data/cx.parquet"]},
+ {"split": "test", "path": "data/bx.parquet"},
+ ],
+ }
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["data/ax.parquet", "data/cx.parquet"]
+
+
+def test_seed_hf_path_keeps_shards_whose_folders_also_hold_other_splits(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [
+ "a/train-0.parquet",
+ "a/test-0.parquet",
+ "b/train-1.parquet",
+ "b/test-1.parquet",
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["a/train-0.parquet", "b/train-1.parquet"]
+
+
+def test_seed_hf_path_refuses_a_mixed_json_split_it_cannot_name(monkeypatch, tmp_path):
+ """*.json* is run over the whole repo, so a a.json.gz would be read too."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/a.json", "data/b.jsonl"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path(
+ "org/repo", files, "train", None, configs, [*files, "data/a.json.gz"]
+ )
+
+ assert resolved is None
+
+
+def test_seed_hf_path_keeps_a_trailing_globstar_a_whole_component(monkeypatch, tmp_path):
+ """fsspec refuses data/**.parquet outright, so the card's data/** grows a /*."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/a/train.parquet", "data/train.parquet"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/**"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ assert resolved == "datasets/org/repo/data/**/*.parquet"
+
+
+def test_seed_hf_path_leaves_hidden_folders_out_of_a_broad_card_glob(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/train.parquet", ".backup/train.parquet"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "**/*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert matched == ["data/train.parquet"]
+
+
+def test_seed_hf_path_reads_a_hidden_folder_the_card_names(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/train.parquet", ".backup/train.parquet"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": ".backup/*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert matched == [".backup/train.parquet"]
+
+
+def test_seed_hf_path_keeps_both_extensions_of_one_builder(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/a.json", "data/b.jsonl"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ assert resolved == "datasets/org/repo/data/*.json*"
+
+
+def test_seed_hf_path_finds_sharded_names_under_a_config_data_dir(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [
+ "scope/data/train-00000-of-00001.parquet",
+ "scope/data/test-00000-of-00001.parquet",
+ "scope/extras/train-extra.parquet",
+ ]
+ configs = [{"config_name": "scoped", "data_dir": "scope"}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", "scoped", configs)
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert matched == ["scope/data/train-00000-of-00001.parquet"]
+
+
+def test_seed_hf_path_refuses_a_declared_split_no_glob_can_name(monkeypatch, tmp_path):
+ """aa and bb cannot be told from ab and ba by any class a glob can carry."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/aa.parquet", "data/ab.parquet", "data/ba.parquet", "data/bb.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": ["data/aa.parquet", "data/bb.parquet"]},
+ {"split": "test", "path": ["data/ab.parquet", "data/ba.parquet"]},
+ ],
+ }
+ ]
+
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) is None
+
+
+def test_seed_hf_path_stops_at_the_sharded_names_the_loader_reads_first(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [
+ "data/train-00000-of-00001.parquet",
+ "data/test-00000-of-00001.parquet",
+ "extras/train-extra.parquet",
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert matched == ["data/train-00000-of-00001.parquet"]
+
+
+ "configs",
+ [
+ [
+ {
+ "config_name": "default",
+ "data_dir": "./data",
+ "data_files": [
+ {"split": "train", "path": "a.parquet"},
+ {"split": "test", "path": "b.parquet"},
+ ],
+ }
+ ],
+ [
+ {
+ "config_name": "default",
+ "data_dir": ".",
+ "data_files": [
+ {"split": "train", "path": "./data/a.parquet"},
+ {"split": "test", "path": "./data/b.parquet"},
+ ],
+ }
+ ],
+ ],
+)
+def test_seed_hf_path_reads_a_card_that_spells_paths_with_a_dot(monkeypatch, tmp_path, configs):
+ """The listing has no ./ in it, so a card carrying one has to lose it."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/a.parquet", "data/b.parquet"]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ assert resolved == "datasets/org/repo/data/a.parquet"
+
+
+ ("split", "expected"),
+ [("Train", "datasets/org/repo/b.parquet"), ("train", "datasets/org/repo/a.parquet")],
+)
+def test_seed_hf_path_reads_a_split_name_exactly(monkeypatch, tmp_path, split, expected):
+ """load_dataset is handed the name the caller asked for, not a folded one."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["a.parquet", "b.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": "a.parquet"},
+ {"split": "Train", "path": "b.parquet"},
+ ],
+ }
+ ]
+
+ assert seed_route._resolve_seed_hf_path("org/repo", files, split, None, configs) == expected
+
+
+def test_seed_hf_path_reads_declared_files_under_data_dir_even_when_they_repeat_it(
+ monkeypatch, tmp_path
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/data/train.parquet", "data/train.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_dir": "data",
+ "data_files": [{"split": "train", "path": "data/train.parquet"}],
+ }
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ assert resolved == "datasets/org/repo/data/data/train.parquet"
+
+
+def test_seed_hf_path_collapses_a_parent_component_in_a_card_path(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/a.parquet", "data/b.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_dir": "staging/../data",
+ "data_files": [
+ {"split": "train", "path": "a.parquet"},
+ {"split": "test", "path": "b.parquet"},
+ ],
+ }
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ assert resolved == "datasets/org/repo/data/a.parquet"
+
+
+def test_seed_hf_path_leaves_loader_ignored_metadata_out_of_a_card_glob(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["train.jsonl", "dataset_info.json"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert matched == ["train.jsonl"]
+
+
+ ("pattern", "expected"),
+ [("[^a].parquet", ["a.parquet"]), ("[!a].parquet", ["b.parquet"])],
+)
+def test_seed_glob_reads_a_caret_in_a_class_literally(monkeypatch, tmp_path, pattern, expected):
+ """fsspec negates on ! alone, so [^a] is the two characters ^ and a."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+
+ assert seed_route._files_under_patterns([pattern], ["a.parquet", "b.parquet"]) == expected
+
+
+def test_seed_hf_path_keeps_a_dunder_file_a_card_declared(monkeypatch, tmp_path):
+ """The loader skips a __folder, not a file whose name happens to start that way."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/__train.json", "data/train.json"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["data/__train.json", "data/train.json"]
+
+
+def test_seed_config_named_Default_is_not_the_implicit_default(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+
+ assert (
+ seed_route._pick_config([{"config_name": "Default"}, {"config_name": "other"}], None)
+ is None
+ )
+
+
+def test_seed_hf_path_ignores_an_extension_collision_out_of_reach(monkeypatch, tmp_path):
+ """The glob is anchored at data/, so a name in other/ cannot spoil it."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/a.json", "data/b.jsonl"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path(
+ "org/repo", files, "train", None, configs, [*files, "other/c.json.gz"]
+ )
+
+ assert resolved == "datasets/org/repo/data/*.json*"
+
+
+ ("subset", "expected"),
+ [
+ ("Foo", "datasets/org/repo/Foo/train*.parquet"),
+ ("foo", "datasets/org/repo/foo/train*.parquet"),
+ ],
+)
+def test_seed_hf_path_narrows_to_the_subset_folder_by_its_own_case(
+ monkeypatch, tmp_path, subset, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["Foo/train.parquet", "foo/train.parquet"]
+
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", subset) == expected
+
+
+def test_seed_format_vote_counts_every_suffix_in_a_name(monkeypatch, tmp_path):
+ """a.csv.parquet is a csv to the loader as much as a parquet."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+
+ assert seed_route._dominant_suffix(["data/a.csv.parquet", "data/b.csv"]) == ".csv"
+ assert seed_route._dominant_suffix(["a.parquet", "b.parquet", "c.json"]) == ".parquet"
+
+
+def test_seed_format_vote_takes_the_window_before_dropping_metadata(monkeypatch, tmp_path):
+ """A shard past the window is one the loader never sees either."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [f"a{i:04d}/metadata.csv" for i in range(200)] + ["z/shard.parquet"]
+
+ assert seed_route._dominant_suffix(files) == ".csv"
+ assert seed_route._dominant_suffix(["a/metadata.csv", "b/shard.parquet"]) == ".parquet"
+
+
+def test_seed_hf_path_unions_an_exact_and_a_qualified_split_folder(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["train/0.parquet", "sets/train_a/1.parquet", "sets/test_b/2.parquet"]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["sets/train_a/1.parquet", "train/0.parquet"]
+
+
+def test_seed_hf_path_keeps_folders_naming_the_split_in_the_middle(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [
+ "sets/a_train_x/0.parquet",
+ "sets/b_train_y/0.parquet",
+ "sets/c_test_z/0.parquet",
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["sets/a_train_x/0.parquet", "sets/b_train_y/0.parquet"]
+
+
+def test_seed_hf_path_keeps_qualified_split_folders_together(monkeypatch, tmp_path):
+ """train_a and train_b are both train to the loader, so both must be read."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [
+ "sets/train_a/0.parquet",
+ "sets/train_b/1.parquet",
+ "sets/test_a/2.parquet",
+ ]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["sets/train_a/0.parquet", "sets/train_b/1.parquet"]
+
+
+def test_seed_hf_path_gathers_a_split_written_under_two_aliases(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/dev-0.parquet", "data/validation-0.parquet", "data/test-0.parquet"]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "validation")
+
+ matched = seed_route._files_under_patterns([resolved[len("datasets/org/repo/") :]], files)
+ assert sorted(matched) == ["data/dev-0.parquet", "data/validation-0.parquet"]
+
+
+def test_seed_hf_path_counts_json_and_jsonl_as_two_extensions(monkeypatch, tmp_path):
+ """`load.infer_module_for_data_files_list` votes per extension, parquet on ties."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/a.json", "data/b.jsonl", "data/c.parquet"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": "data/*"}]}]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+
+ assert resolved == "datasets/org/repo/data/*.parquet"
+
+
+def test_seed_hf_path_keeps_shards_spread_over_sibling_folders(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["a/train-0.parquet", "b/train-1.parquet", "a/test-0.parquet"]
+
+ resolved = seed_route._resolve_seed_hf_path("org/repo", files, "train")
+
+ assert resolved == "datasets/org/repo/**/train-*.parquet"
+
+
+def test_seed_hf_path_reads_sibling_folders_as_one_split_without_a_card(monkeypatch, tmp_path):
+ """Without a card there are no configs, so every train file is the train split."""
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [
+ "main/train-0.parquet",
+ "main/test-0.parquet",
+ "socratic/train-0.parquet",
+ "socratic/test-0.parquet",
+ ]
+
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train")
+ == "datasets/org/repo/**/train-*.parquet"
+ )
+ # Ask for one of them and only that one is read.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "socratic")
+ == "datasets/org/repo/socratic/train-*.parquet"
+ )
+
+
+def test_seed_preview_file_follows_the_config_data_dir(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["en/train.parquet", "fr/train.parquet"]
+ configs = [{"config_name": "french", "data_dir": "fr"}]
+ monkeypatch.setattr(seed_route, "_list_hf_repo_files", lambda **kwargs: files)
+ monkeypatch.setattr(seed_route, "_list_hf_dataset_configs", lambda **kwargs: configs)
+ monkeypatch.setattr(
+ seed_route, "refuse_unauthorized_dataset_preview", lambda *args, **kwargs: None
+ )
+ seen: list[str | None] = []
+
+ def fake_preview(*, load_dataset_fn, load_kwargs, preview_size):
+ seen.append((load_kwargs.get("data_files") or [None])[0])
+ return [{"text": "row"}]
+
+ monkeypatch.setattr(seed_route, "_load_preview_rows", fake_preview)
+
+ response = seed_route.inspect_seed_dataset(
+ SimpleNamespace(
+ dataset_name = "org/repo",
+ split = "train",
+ subset = "french",
+ hf_token = None,
+ preview_size = 1,
+ ),
+ allow_ambient_token = False,
+ )
+
+ assert seen[0] == "fr/train.parquet"
+ assert response.resolved_path == "datasets/org/repo/fr/train*.parquet"
+
+
+def test_seed_hf_path_treats_an_unnamed_config_as_the_default(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ configs = [
+ {
+ "data_files": [
+ {"split": "train", "path": "a/part.parquet"},
+ {"split": "test", "path": "b/part.parquet"},
+ ]
+ }
+ ]
+ files = ["a/part.parquet", "b/part.parquet"]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "test", "default", configs)
+ == "datasets/org/repo/b/part.parquet"
+ )
+
+
+def test_seed_hf_path_ignores_a_subset_label_that_is_not_the_config(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ # x/main.parquet carries the label but nothing of the split, so it is a name
+ # collision rather than the config.
+ files = ["x/main.parquet", "y/train-0.parquet"]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "main")
+ == "datasets/org/repo/y/train-*.parquet"
+ )
+
+
+def test_seed_hf_path_keeps_the_split_when_widening_several_declared_globs(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["sets/a/train-0.parquet", "sets/b/train-0.parquet", "sets/a/test-0.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ # Declared twice for the same split, which must not lose the second.
+ {"split": "train", "path": "sets/a/train-*"},
+ {"split": "train", "path": "sets/b/train-*"},
+ ],
+ }
+ ]
+ assert seed_route._declared_split_patterns(configs, "train") == [
+ "sets/a/train-*",
+ "sets/b/train-*",
+ ]
+ # sets/**/*.parquet would take the test file back; the split-named form does not.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/sets/**/*train*.parquet"
+ )
+
+
+def test_seed_hf_path_keeps_a_split_whose_name_carries_a_separator(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = [
+ "data/validation_matched-000.parquet",
+ "data/validation_mismatched-000.parquet",
+ "data/train-000.parquet",
+ ]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "validation_matched")
+ == "datasets/org/repo/data/validation_matched-*.parquet"
+ )
+
+
+def test_seed_hf_path_keeps_a_subset_whose_name_carries_a_separator(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["sample-10BT-train.parquet", "sample-100BT-train.parquet"]
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", "sample-10BT")
+ == "datasets/org/repo/sample-10BT-train*.parquet"
+ )
+
+
+def test_seed_hf_path_widens_through_split_named_folders(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["sets/train_a/0.parquet", "sets/train_b/0.parquet", "sets/test/0.parquet"]
+ configs = [
+ {
+ "config_name": "default",
+ "data_files": [
+ {"split": "train", "path": ["sets/train_a/*", "sets/train_b/*"]},
+ {"split": "test", "path": "sets/test/*"},
+ ],
+ }
+ ]
+ # The shard names say nothing, so the split has to come from the folders.
+ assert (
+ seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs)
+ == "datasets/org/repo/sets/**/*train*/**/*.parquet"
+ )
+
+
+def test_seed_globstar_keeps_the_folder_boundary(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/train.csv", "data/z/train.csv", "data/nottrain.parquet"]
+ assert seed_route._files_under_patterns(["data/**/train.*"], files) == [
+ "data/train.csv",
+ "data/z/train.csv",
+ ]
+
+
+def test_seed_declared_files_match_the_glob_not_its_prefix(monkeypatch, tmp_path):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/train-notes.json", "data/train-00000-of-00002.parquet"]
+ assert seed_route._files_under_patterns(["data/train-*.parquet"], files) == [
+ "data/train-00000-of-00002.parquet"
+ ]
+ assert seed_route._files_under_patterns(["data/*/x.parquet"], ["data/a/b/x.parquet"]) == []
+ assert seed_route._files_under_patterns(["data/**/x.parquet"], ["data/a/b/x.parquet"]) == [
+ "data/a/b/x.parquet"
+ ]
+
+
+ ("files", "expected"),
+ [
+ # train-part.parquet wins on length, but questions_train.parquet is train
+ # too, so the pattern has to reach both without taking test-part.
+ (
+ ["data/train-part.parquet", "data/questions_train.parquet", "data/test-part.parquet"],
+ "datasets/org/repo/data/*train*.parquet",
+ ),
+ # A split named mid-name, the conventional sharded form.
+ (
+ ["data/questions_train_000.jsonl", "data/questions_test_000.jsonl"],
+ "datasets/org/repo/data/*train*.jsonl",
+ ),
+ # A dotted split name: only the final extension comes off the stem.
+ (
+ ["data/questions.train.parquet", "data/questions.test.parquet"],
+ "datasets/org/repo/data/*train*.parquet",
+ ),
+ # "training" is datasets' own alias for train, so both belong to it.
+ (
+ ["data/train-0.parquet", "data/training-0.parquet", "data/test-0.parquet"],
+ "datasets/org/repo/data/*train*.parquet",
+ ),
+ ],
+)
+def test_seed_hf_path_reaches_every_file_of_the_split_and_no_other(
+ monkeypatch, tmp_path, files, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train") == expected
+
+
+ ("declared", "expected"),
+ [
+ ("data/train.*", "datasets/org/repo/data/train.parquet"),
+ ("data/train-*.*", "datasets/org/repo/data/train-*.parquet"),
+ ("data/train-*", "datasets/org/repo/data/train-*.parquet"),
+ ("data/train-0.parquet", "datasets/org/repo/data/train-0.parquet"),
+ ],
+)
+def test_seed_hf_path_gives_a_card_glob_a_readable_extension(
+ monkeypatch, tmp_path, declared, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ files = ["data/train.parquet", "data/train-0.parquet", "data/test-0.parquet"]
+ configs = [{"config_name": "default", "data_files": [{"split": "train", "path": declared}]}]
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train", None, configs) == expected
+
+
+ ("files", "expected"),
+ [
+ (["part-0.parquet"], "datasets/org/repo/**/*.parquet"),
+ (["data/part-0.parquet", "data/part-1.parquet"], "datasets/org/repo/data/**/*.parquet"),
+ (
+ ["data/test/0.parquet", "data/train/0.parquet"],
+ "datasets/org/repo/data/train/**/*.parquet",
+ ),
+ # Split folders at the repo root: the shorter "test" path used to win on
+ # length alone, so a train recipe was served the test split.
+ (["test/0.parquet", "train/0.parquet"], "datasets/org/repo/train/**/*.parquet"),
+ ],
+)
+def test_seed_hf_path_still_globs_the_directory_without_a_split_in_the_name(
+ monkeypatch, tmp_path, files, expected
+):
+ seed_route = _load_seed_route(monkeypatch, tmp_path)
+ assert seed_route._resolve_seed_hf_path("org/repo", files, "train") == expected
+
+
def test_validate_resolves_the_hf_seed_endpoint_like_jobs(monkeypatch):
pytest.importorskip("fastapi")
backend_root = Path(__file__).resolve().parent.parent
diff --git a/studio/backend/tests/test_hub_token_caller_identity.py b/studio/backend/tests/test_hub_token_caller_identity.py
index 29be59bca81..a888de1aa96 100644
--- a/studio/backend/tests/test_hub_token_caller_identity.py
+++ b/studio/backend/tests/test_hub_token_caller_identity.py
@@ -834,7 +834,7 @@ def test_seed_inspection_derives_its_policy_from_the_caller(monkeypatch):
seen["token"] = token
return []
- monkeypatch.setattr(seed_routes, "_list_hf_data_files", _fake_list)
+ monkeypatch.setattr(seed_routes, "_list_hf_repo_files", _fake_list)
for via_api_key in (True, False):
_router_client(seed_routes.router, "/api/data-recipe", via_api_key = via_api_key).post(
@@ -852,7 +852,7 @@ def test_an_explicit_seed_token_wins_for_either_caller(monkeypatch):
_counting_probe(monkeypatch, True)
monkeypatch.setattr(
seed_routes,
- "_list_hf_data_files",
+ "_list_hf_repo_files",
lambda *, dataset_name, token: seen.update(token = token) or [],
)
monkeypatch.setattr(seed_routes, "load_dataset", lambda **_k: iter([]), raising = False)
@@ -1362,7 +1362,7 @@ def test_a_seed_preview_off_the_cache_is_refused(monkeypatch, reason, hf_token,
def _never(*_a, **_k):
raise AssertionError(f"{reason} reached the dataset load")
- monkeypatch.setattr(seed_routes, "_list_hf_data_files", _never)
+ monkeypatch.setattr(seed_routes, "_list_hf_repo_files", _never)
payload = SimpleNamespace(
dataset_name = "org/private",