Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

69.3 KB
Raw Blame
"""Data loading from local files and HuggingFace."""

from __future__ import annotations

import json
import os
from dataclasses import dataclass, replace
from functools import partial
from itertools import islice
from pathlib import Path

from rich.console import Console

from soup_cli.config.schema import DataConfig
from soup_cli.data.formats import (
    MAX_DETECT_ROWS,
    detect_format,
    format_to_messages_with_reason,
    is_audio_format,
    is_vision_format,
    unify_preference_layout,
)
from soup_cli.data.validator import _MAX_REASON_SAMPLES
from soup_cli.utils.paths import is_under_cwd

console = Console()

# File extensions we support
SUPPORTED_EXTENSIONS = {".jsonl", ".json", ".csv", ".parquet", ".txt"}

# HF `datasets` builder name per suffix, for _load_interleaved_streaming_
# datasets (#459/#468 MEDIUM fix) — mirrors SUPPORTED_EXTENSIONS /
# load_raw_data's per-suffix dispatch so streaming reads the same file
# type the non-streaming local loader would, instead of always assuming
# JSON. Keys match SUPPORTED_EXTENSIONS exactly (one entry each).
_STREAMING_BUILDERS = {
    ".jsonl": "json",
    ".json": "json",
    ".csv": "csv",
    ".parquet": "parquet",
    ".txt": "text",
}


def _streaming_builder_for(entry: str) -> str:
    """Pick the HF ``datasets`` builder for one streaming interleave entry
    by file suffix, so flipping only ``data.streaming: true`` doesn't
    silently misparse a csv/txt/parquet file as JSON (#459/#468 MEDIUM).

    Call with the CANONICALIZED entry (post ``validate_remote_uri``), not
    the raw one — a query string surviving on the raw string could
    otherwise be mistaken for the suffix.
    """
    suffix = Path(entry).suffix.lower()
    try:
        return _STREAMING_BUILDERS[suffix]
    except KeyError:
        raise ValueError(
            f"data.streaming=true does not support '{suffix or '(no extension)'}' "
            f"data.train entries ({entry!r}) -- supported: "
            f"{sorted(_STREAMING_BUILDERS)}"
        ) from None


# Cap on rows materialised from a remote/streaming source — matches v0.24.0
# ``soup data download --samples`` ceiling. Defends against OOM when a
# crafted / oversized bucket object or hub dataset is pointed at via
# streaming + eager-materialise. Shared by _load_remote_dataset (v0.53.8
# #85) and _load_interleaved_streaming_datasets (#459) so the ceiling can't
# drift between the two streaming call sites.
MAX_REMOTE_ROWS = 1_000_000


def load_raw_data(
    path: Path, *, issues: list[str] | None = None, skip_non_object: bool = True,
) -> list[dict]:
    """Load file rows, skipping non-objects unless a caller needs the original JSON rows.

    ``issues`` receives at most ``_MAX_REASON_SAMPLES`` messages per kind of skipped row.
    Pass a ``SkippedRows`` to also get the exact number of skipped rows in ``.total``.
    """
    if not path.exists():
        raise FileNotFoundError(f"Data file not found: {path}")

    ext = path.suffix.lower()
    if ext not in SUPPORTED_EXTENSIONS:
        raise ValueError(f"Unsupported file format: {ext}. Supported: {SUPPORTED_EXTENSIONS}")

    if ext == ".jsonl":
        return _load_jsonl(path, issues=issues, skip_non_object=skip_non_object)
    elif ext == ".json":
        return _load_json(path, issues=issues, skip_non_object=skip_non_object)
    elif ext == ".csv":
        return _load_csv(path)
    elif ext == ".parquet":
        return _load_parquet(path)
    elif ext == ".txt":
        return _load_txt(path)

    raise ValueError(f"Unsupported format: {ext}")


def non_object_row_issue(value: object, position: int, unit: str = "line") -> str:
    """Describe a rejected JSON value without printing its possibly sensitive contents."""
    kind = {
        str: "string", list: "array", int: "number", float: "number",
        bool: "boolean", type(None): "null",
    }[type(value)]
    return f"{unit} {position} is a JSON {kind}, not an object"


class SkippedRows(list):
    """``issues`` list for ``load_raw_data``: keeps a bounded sample plus an exact total."""

    def __init__(self) -> None:
        super().__init__()
        self.total = 0


class SkipTally:
    """Counts rows skipped for one reason; prints only the first few warnings.

    The exact ``count`` is always kept. Warnings, and the messages recorded in
    ``issues``, stop after ``_MAX_REASON_SAMPLES``; ``finish()`` prints one line
    for the rest.
    """

    def __init__(
        self, summary: str, *, issues: list[str] | None = None, record: bool = False,
    ) -> None:
        self.summary = summary
        self.count = 0
        self.issues = issues
        self.record = record

    def add(self, message: str) -> None:
        self.count += 1
        if self.record and isinstance(self.issues, SkippedRows):
            self.issues.total += 1
        if self.count > _MAX_REASON_SAMPLES:
            return
        console.print(f"[yellow]Warning: {message}[/]")
        if self.record and self.issues is not None:
            self.issues.append(message)

    def finish(self) -> None:
        extra = self.count - _MAX_REASON_SAMPLES
        if extra > 0:
            console.print(f"[yellow]Warning: ... and {extra} more {self.summary}[/]")


def _load_jsonl(
    path: Path, *, issues: list[str] | None = None, skip_non_object: bool = True,
) -> list[dict]:
    data = []
    non_objects = SkipTally(
        "lines skipped (JSON values that are not objects)", issues=issues, record=True,
    )
    bad_json = SkipTally("lines skipped (invalid JSON)", issues=issues, record=True)
    # v0.40.1 Part E — auto-strip UTF-8 BOM (Windows users overwhelmingly
    # write JSONL via PowerShell `Out-File -Encoding utf8` which adds BOM).
    # The ``utf-8-sig`` codec consumes the BOM transparently if present.
    with open(path, encoding="utf-8-sig") as f:
        for i, line in enumerate(f):
            line = line.strip()
            if not line:
                continue
            try:
                row = json.loads(line)
                if skip_non_object and not isinstance(row, dict):
                    non_objects.add(non_object_row_issue(row, i + 1))
                    continue
                data.append(row)
            except json.JSONDecodeError as e:
                bad_json.add(f"invalid JSON on line {i + 1}: {e}")
    non_objects.finish()
    bad_json.finish()
    return data


def _load_json(
    path: Path, *, issues: list[str] | None = None, skip_non_object: bool = True,
) -> list[dict]:
    with open(path, encoding="utf-8-sig") as f:
        raw = json.load(f)
    if isinstance(raw, list):
        if not skip_non_object:
            return raw
        rows = []
        non_objects = SkipTally(
            "rows skipped (JSON values that are not objects)", issues=issues, record=True,
        )
        for position, row in enumerate(raw, start=1):
            if not isinstance(row, dict):
                non_objects.add(non_object_row_issue(row, position, "row"))
                continue
            rows.append(row)
        non_objects.finish()
        return rows
    raise ValueError("JSON file must contain a list of objects")


def _load_csv(path: Path) -> list[dict]:
    import csv

    with open(path, encoding="utf-8-sig") as f:
        reader = csv.DictReader(f)
        return list(reader)


def _load_parquet(path: Path) -> list[dict]:
    try:
        import pyarrow.parquet as pq
    except ImportError:
        raise ImportError("Install pyarrow to read parquet files: pip install pyarrow")

    table = pq.read_table(path)

    # pandas stores an unnamed, non-default index (after df.sample() or a filter)
    # as a hidden __index_level_N__ column; pd.read_parquet restored it as the index.
    pandas_meta = table.schema.pandas_metadata or {}
    hidden = {
        name
        for name in pandas_meta.get("index_columns", [])
        if isinstance(name, str) and name.startswith("__index_level_")
    }

    if hidden:
        table = table.select(
            [name for name in table.column_names if name not in hidden]
        )
    return table.to_pylist()


def _load_txt(path: Path) -> list[dict]:
    """Load a plain text file as a list of {text: ...} dicts.

    Each non-empty line is treated as a separate document.
    Empty lines are skipped.
    """
    file_size = path.stat().st_size
    if file_size > 500 * 1024 * 1024:  # 500 MB
        console.print(
            f"[yellow]Warning: large text file ({file_size / 1024 / 1024:.0f} MB). "
            f"Consider splitting into smaller files or using JSONL format.[/]"
        )
    with open(path, encoding="utf-8-sig") as f:
        content = f.read()

    # Split by double newline (paragraph/document separator) or treat each line as a doc
    lines = [line.strip() for line in content.split("\n") if line.strip()]
    if not lines:
        console.print(f"[yellow]Warning: empty text file: {path}[/]")
        return []

    return [{"text": line} for line in lines]


@dataclass(frozen=True)
class LoadOutcome:
    """What the most recent :func:`load_dataset` call converted (#1217).

    ``fmt`` is the format the rows were converted as (resolved, never
    ``"auto"``); ``first_drop`` is ``(format, row index, reason, source)`` for
    the first row a converter dropped. ``soup train`` reads it to say why a
    load ended with zero training rows instead of printing "Ready to train".

    Frozen, and replaced (never mutated) as a load progresses, so a caller
    holding one keeps a snapshot that no later load can change.
    """

    fmt: str | None = None
    first_drop: tuple[str, int, str, str | None] | None = None


_last_load = LoadOutcome()


def last_load_outcome() -> LoadOutcome:
    """The :class:`LoadOutcome` of the most recent :func:`load_dataset` call."""
    return _last_load


# Tasks that require source columns preserved across dataset format normalisation (#1219).
# GRPO uses raw columns for custom reward functions (answer, expected, etc.).
# The classifier family (classifier, reranker, cross_encoder) uses source columns
# to retain label, paired text (text_a, text_b, question, answer), and metadata.
PRESERVE_SOURCE_TASKS: frozenset[str] = frozenset({
    "grpo",
    "classifier",
    "reranker",
    "cross_encoder",
})


def task_preserves_source_columns(task: str) -> bool:
    """Return whether task requires source dataset columns to be preserved."""
    return task in PRESERVE_SOURCE_TASKS


PREFERENCE_PAIR_TASKS = frozenset({
    "dpo", "ipo", "simpo", "orpo", "bco", "preference", "reward_model",
})


def data_config_for_task(data_config: DataConfig, task: str) -> DataConfig:
    """Resolve task-specific ``format: auto`` without changing the input config.

    ``detect_format`` cannot see the task, and ``{question, answer}`` is also
    GSM8K's shape, so pair rows are recognised only when the task reads them.
    """
    if task == "cross_encoder" and data_config.format == "auto":
        return data_config.model_copy(update={"format": "cross_encoder"})
    if task in PREFERENCE_PAIR_TASKS and data_config.format == "auto":
        return data_config.model_copy(update={"format": "dpo"})
    return data_config


def _format_rows(
    raw_data: list[dict],
    fmt: str,
    *,
    preserve_source_columns: bool = False,
    source: str | None = None,
    kept_indices: list[int] | None = None,
) -> list[dict]:
    """Normalize rows, optionally retaining columns used by GRPO rewards.

    TRL forwards every non-prompt dataset column to custom reward functions.
    Soup's normalizers intentionally return only task-specific columns, which
    is correct for supervised and preference trainers but would discard GRPO
    references such as ``answer``, ``expected``, ``schema``, or custom
    metadata.  The opt-in keeps the default loader contract byte-for-byte
    unchanged for every other task.

    ``kept_indices``, when given, receives the ``raw_data`` index of every row
    that converted, in order, so a caller can map a formatted row back to the
    raw row it came from.
    """
    formatted: list[dict] = []
    dropped = 0
    first_drop: tuple[int, str] | None = None
    for index, raw_row in enumerate(raw_data):
        # #1181: the same drop decision as format_to_messages, plus the reason,
        # so a dropped row is counted and named instead of vanishing.
        normalized, reason = format_to_messages_with_reason(raw_row, fmt)
        if normalized is None:
            dropped += 1
            if first_drop is None:
                first_drop = (index, reason or "")
            continue
        if preserve_source_columns:
            normalized = {**raw_row, **normalized}
        formatted.append(normalized)
        if kept_indices is not None:
            kept_indices.append(index)
    global _last_load
    if dropped and first_drop is not None:
        _report_dropped_rows(dropped, len(raw_data), fmt, first_drop, source)
        if _last_load.first_drop is None:
            _last_load = replace(
                _last_load, first_drop=(fmt, first_drop[0], first_drop[1], source)
            )
    if _last_load.fmt is None:
        _last_load = replace(_last_load, fmt=fmt)
    return formatted


def _report_dropped_rows(
    dropped: int,
    total: int,
    fmt: str,
    first_drop: tuple[int, str],
    source: str | None,
) -> None:
    """One warning for the rows the format converter dropped (#1181).

    The drop is the loader's documented contract (one bad line must not abort a
    load); the silence was the defect. Same shape as the media paths' "rows
    skipped" lines, with the first row's index and the converter's own reason.
    """
    from rich.markup import escape

    index, reason = first_drop
    message = (
        f"[yellow]Warning: {dropped} of {total} rows dropped: they do not convert "
        f"as {escape(repr(fmt))}. First: row {index}: {escape(reason)}.[/]"
    )
    if source:
        message += (
            f"\n  List them all with: soup data validate {escape(source)} --format "
            f"{escape(fmt)}"
        )
    console.print(message)


def _load_replay_rows(
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> list[dict]:
    """Load + normalize the replay file with its OWN format detection.

    The old dataset may be alpaca while the new one is sharegpt, so the
    replay file cannot inherit ``data_config.format``.

    It also gets its OWN media-containment pass. ``load_dataset`` runs
    :func:`_validate_vision_images` / :func:`_validate_audio_files` on the
    primary dataset, and the replay file is loaded here rather than there —
    so without this, a llava-shaped replay row's ``image`` value survives
    ``format_to_messages`` untouched and a traversal path would reach
    ``PIL.Image.open`` in the trainer. Media resolve against the REPLAY
    file's own directory unless an explicit dir is configured: the old
    dataset's images live with the old dataset.
    """
    replay_path = Path(data_config.replay)
    if not is_under_cwd(replay_path):
        raise ValueError(
            f"data.replay path is outside the working directory: {replay_path}"
        )
    if not replay_path.exists():
        raise FileNotFoundError(
            f"data.replay file not found: {replay_path}"
        )
    raw = load_raw_data(replay_path)
    fmt = detect_format(raw)
    rows = _format_rows(
        raw,
        fmt,
        preserve_source_columns=preserve_source_columns,
        source=str(replay_path),
    )

    if is_vision_format(fmt):
        image_dir = (
            Path(data_config.image_dir)
            if data_config.image_dir
            else replay_path.parent
        )
        rows = _validate_vision_images(rows, image_dir)
    if is_audio_format(fmt):
        audio_dir = (
            Path(data_config.audio_dir)
            if data_config.audio_dir
            else replay_path.parent
        )
        rows = _validate_audio_files(rows, audio_dir)
    return rows


def _finalize(
    formatted: list[dict],
    data_config: DataConfig,
    *,
    val: list[dict] | None = None,
    preserve_source_columns: bool = False,
) -> dict:
    """Split train/val, then mix replay into train ONLY.

    Single exit point for every load path (local / remote / HF) so replay
    behaviour cannot drift between them — `soup sweep` and
    `soup train --dry-run` go through the same seam.

    Replay is mixed AFTER the split so val stays pure new-task: it is the
    yardstick for the task being learned. Old-task retention is measured
    externally with `soup eval custom` / `soup ship`, which adds no new
    eval machinery here.
    """
    if val is not None:
        result = {"train": formatted, "val": val}
    elif data_config.val_split > 0:
        train_rows, val_rows = _split_val(formatted, data_config.val_split)
        result = {"train": train_rows, "val": val_rows}
    else:
        result = {"train": formatted}

    if getattr(data_config, "replay", None):
        from soup_cli.utils.rehearsal import mix_replay

        replay_rows = _load_replay_rows(
            data_config,
            preserve_source_columns=preserve_source_columns,
        )
        mixed, report = mix_replay(
            result["train"],
            replay_rows,
            ratio=data_config.replay_ratio,
            seed=data_config.replay_seed,
        )
        result["train"] = mixed
        console.print(
            f"[dim]Replay: +{report.n_replay} old rows interleaved "
            f"({report.ratio_actual * 100:.1f}% of {report.n_final})[/]"
        )
        if report.shortfall:
            console.print(
                f"[yellow]Replay pool too small: wanted {report.requested}, "
                f"used {report.n_replay} (short {report.shortfall}). Rows are "
                "NOT repeated.[/]"
            )

    # #1849: text and conversational preference rows cannot share a dataset
    # (one Arrow type per column; trl reads the layout off the first row).
    # Here rather than per file, so interleaved files in different layouts are
    # covered too. trl prepares train and val separately.
    for split in ("train", "val"):
        rows = result.get(split)
        if not isinstance(rows, list):
            continue
        unified, rewritten = unify_preference_layout(rows)
        if rewritten:
            result[split] = unified
            what, each = (
                ("is a plain-text preference row", "It")
                if rewritten == 1
                else ("are plain-text preference rows", "Each")
            )
            console.print(
                f"[yellow]Note: {rewritten} of {len(unified)} {split} rows {what} in a "
                f"dataset whose other rows are conversations. {each} was rewritten as one "
                "user turn and one assistant turn per answer, so the whole split goes "
                "through the chat template.[/]"
            )
    return result


def _classify_train_entry(value: str) -> str:
    """Classify one data.train entry: a list element (#459) or the single string.

    Returns ``'remote'`` / ``'hub'`` / ``'local'``. Mirrors the identical
    classification in schema._classify_data_train_entry — kept
    as two copies of the same three-line rule (suffix-in-SUPPORTED_EXTENSIONS
    check + "://"-in-entry check) rather than one shared function, since
    schema.py must stay import-light (no torch-adjacent deps) and loader.py
    already owns _looks_like_remote_uri; the rule itself, not the function,
    is the single source of truth, and both sites are covered by
    tests/test_issue459_interleave_streaming_hub.py's schema+loader pairs.

    A suffix must be one of SUPPORTED_EXTENSIONS to count as 'local' (#468
    review fix) — "any non-empty Path.suffix" previously misclassified any
    hub name with a version number (``teknium/OpenHermes-2.5``,
    ``mlfoundations/dclm-baseline-1.0``: Path.suffix is ``.5`` / ``.0``) as
    a local file, even though the hub-loading path itself handles these
    names correctly once actually dispatched there.

    Any ``"://"``-bearing entry classifies 'remote' regardless of scheme
    (#468 review fix) — the scheme allowlist is enforced downstream by
    validate_remote_uri, which refuses a non-allowlisted scheme BY NAME;
    classifying only allowlisted schemes as 'remote' let e.g. an
    ``https://...`` entry with a familiar suffix fall through to 'local'
    and reach hf_load unvalidated instead.
    """
    if _looks_like_remote_uri(value):
        return "remote"
    if Path(value).suffix.lower() in SUPPORTED_EXTENSIONS:
        return "local"
    return "hub"


def load_dataset(
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> dict:
    """Load dataset for training. Returns dict with 'train' and optionally 'val' keys.

    Supports:
    - Local files (.jsonl, .json, .csv, .parquet, .txt)
    - HuggingFace dataset names (any string that is neither a local file nor a
      remote URI; a dot in the name, as in ``teknium/OpenHermes-2.5``, does not
      make it a file)
    - Remote fsspec URIs (s3://, gs://, gcs://, az://, abfs://, abfss://, oci://) — v0.53.8 #85
    - A list of >= 2 local files / remote URIs combined via data.interleave
      (#443 local files; #459 widened to remote URIs when data.streaming is
      set, and to a separate all-HF-hub-dataset-name list shape). Schema
      validation (SoupConfig._validate_interleave_compat) already
      guarantees data.interleave is set and every entry classifies
      consistently whenever data.train is a list, so this branch only has
      to pick which loader — not re-validate the shape.
    """
    global _last_load
    _last_load = LoadOutcome()

    train_path = data_config.train

    if isinstance(train_path, list):
        kinds = {_classify_train_entry(p) for p in train_path}
        if kinds == {"hub"}:
            return _load_interleaved_hub_datasets(
                train_path,
                data_config,
                preserve_source_columns=preserve_source_columns,
            )
        return _load_interleaved_local_datasets(
            train_path,
            data_config,
            preserve_source_columns=preserve_source_columns,
        )

    # v0.53.8 #85 — fsspec live remote loader. Schema accepts these URIs
    # since v0.42.0; live loader lands here. Lazy-imports fsspec + the
    # backend driver (s3fs / gcsfs / adlfs / ocifs) and surfaces a
    # friendly Rich panel naming the pip install when the driver is
    # missing.
    if _looks_like_remote_uri(train_path):
        return _load_remote_dataset(
            train_path,
            data_config,
            preserve_source_columns=preserve_source_columns,
        )

    # Hub id or local file: the list form's rule, so one dataset name takes the
    # same route in both spellings (a dot in ``teknium/OpenHermes-2.5`` is not a
    # file extension). One addition: a string with a suffix that is an existing
    # file stays local, so a real ``data.tsv`` still gets "Unsupported file
    # format" instead of a Hub lookup.
    if _classify_train_entry(train_path) == "hub" and not (
        Path(train_path).suffix and os.path.isfile(train_path)
    ):
        return _load_hf_dataset(
            train_path,
            data_config,
            preserve_source_columns=preserve_source_columns,
        )

    # Local file
    path = Path(train_path)
    raw_data = load_raw_data(path)

    # Detect or use specified format
    fmt = data_config.format
    if fmt == "auto":
        fmt = detect_format(raw_data)
        console.print(f"[dim]Auto-detected format: {fmt}[/]")

    # Convert to standard message format
    formatted = _format_rows(
        raw_data,
        fmt,
        preserve_source_columns=preserve_source_columns,
        source=str(path),
    )

    # Validate image paths for vision formats
    if is_vision_format(fmt):
        image_dir = Path(data_config.image_dir) if data_config.image_dir else path.parent
        formatted = _validate_vision_images(formatted, image_dir)

    # Validate audio paths for audio formats
    if is_audio_format(fmt):
        audio_dir = Path(data_config.audio_dir) if data_config.audio_dir else path.parent
        formatted = _validate_audio_files(formatted, audio_dir)

    # Split into train/val, then mix replay into train (v0.71.36).
    return _finalize(
        formatted,
        data_config,
        preserve_source_columns=preserve_source_columns,
    )


def _load_one_local_dataset(
    train_path: str,
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> list[dict]:
    """Load + format one local file — the same per-file pipeline the
    single-path branch of load_dataset() runs above, factored out so
    #443's interleave path (below) reuses it rather than re-deriving it.
    The single-path branch itself is left inline and untouched, so its
    output stays byte-identical by construction rather than by
    refactor-equivalence.
    """
    path = Path(train_path)
    raw_data = load_raw_data(path)

    fmt = data_config.format
    if fmt == "auto":
        fmt = detect_format(raw_data)
        console.print(f"[dim]Auto-detected format ({train_path}): {fmt}[/]")

    formatted = _format_rows(
        raw_data,
        fmt,
        preserve_source_columns=preserve_source_columns,
        source=str(path),
    )

    if is_vision_format(fmt):
        image_dir = Path(data_config.image_dir) if data_config.image_dir else path.parent
        formatted = _validate_vision_images(formatted, image_dir)
    if is_audio_format(fmt):
        audio_dir = Path(data_config.audio_dir) if data_config.audio_dir else path.parent
        formatted = _validate_audio_files(formatted, audio_dir)
    return formatted


def _split_val(rows: list[dict], val_split: float) -> tuple[list[dict], list[dict]]:
    """Slice ``rows`` into (train, val) at the same fraction _finalize has
    always used, factored out so callers can carve val out of a row set
    that has not been through interleave oversampling yet (#680).
    """
    split_idx = int(len(rows) * (1 - val_split))
    return rows[:split_idx], rows[split_idx:]


def _split_val_per_source(
    rows: list[dict], val_split: float, strategy: str
) -> tuple[list[dict], list[dict]]:
    """``_split_val`` for a per-source carve-out (#680). An empty train side
    here would otherwise surface downstream as _combine_interleaved's generic
    "must have >= 1 row" error, which is misleading: the source did have
    rows, val_split just consumed all of them.
    """
    train_rows, val_rows = _split_val(rows, val_split)
    if rows and not train_rows:
        raise ValueError(
            f"data.interleave: data.val_split={val_split} leaves 0 training "
            f"rows for a source with {len(rows)} row(s) under strategy "
            f"'{strategy}', which carves val out per source. Use "
            "'concat'/'under', lower val_split, or drop the source."
        )
    return train_rows, val_rows


def _row_key(row: dict) -> str:
    """A stable identity for a formatted row, for duplicate detection.

    Rows are plain dicts of JSON-ish values, so a sorted-key dump is both
    hashable and order-insensitive. Falls back to ``repr`` for anything
    ``json`` refuses, which keeps an exotic value from turning a leak check
    into a crash.
    """
    import json

    try:
        return json.dumps(row, sort_keys=True, default=repr)
    except (TypeError, ValueError):
        return repr(sorted(row.items(), key=lambda kv: str(kv[0])))


def _split_val_deduplicated(
    rows: list[dict], val_split: float
) -> tuple[list[dict], list[dict]]:
    """Carve val out of ``rows`` so no val row has a duplicate left in train (#702).

    The streaming ``over`` path interleaves with
    ``stopping_strategy="all_exhausted"``, which recycles the shorter stream
    to exhaust the longer one. A row and its recycled copy are separate
    entries, so the positional tail slice ``_split_val`` takes could put one
    on each side -- validation loss on the recycled source then measures
    memorisation and reports a suspiciously good number rather than an error.

    A stream is not countable ahead of time, so #701's per-source carve-out
    does not transfer. The split is instead sized over *distinct* rows, and
    val is drawn from the tail of the distinct rows in stream order,
    preferring content that occurs exactly once: such a row has no copy to
    leave behind, so taking it costs train nothing. Content is keyed by
    value, so a recycled copy and a genuinely repeated source row are
    indistinguishable; only when too few unique rows exist is repeated
    content used, and then every other copy is withheld from train and the
    number withheld is reported rather than dropped silently.

    Raises ``ValueError`` when the split would leave no training rows, the
    same refusal ``_split_val_per_source`` makes on the eager path.
    """
    from collections import Counter

    keys = [_row_key(row) for row in rows]
    counts = Counter(keys)
    first_index: dict[str, int] = {}
    for index, key in enumerate(keys):
        first_index.setdefault(key, index)
    distinct = list(first_index)

    n_val = len(distinct) - int(len(distinct) * (1 - val_split))
    newest_first = distinct[::-1]
    unique = [key for key in newest_first if counts[key] == 1]
    repeated = [key for key in newest_first if counts[key] > 1]
    chosen = unique[:n_val]
    fallback = repeated[: n_val - len(chosen)]
    val_keys = set(chosen) | set(fallback)

    val_rows = [rows[first_index[key]] for key in sorted(val_keys, key=first_index.get)]
    train_rows = [row for row, key in zip(rows, keys) if key not in val_keys]

    if rows and not train_rows:
        raise ValueError(
            f"data.interleave: data.val_split={val_split} leaves 0 training rows "
            f"under streaming 'over', which splits over distinct rows: the "
            f"{len(rows)} materialised row(s) hold {len(distinct)} distinct "
            "row(s). Add distinct rows, set val_split: 0, or use 'concat'/'under'."
        )
    withheld = sum(counts[key] - 1 for key in fallback)
    if withheld:
        console.print(
            f"[yellow]Warning: data.val_split under streaming 'over' withheld "
            f"{withheld} duplicate row(s) from train: too few rows with unique "
            f"content to fill val, so {len(fallback)} val row(s) have copies "
            "that would otherwise have leaked into train.[/]"
        )
    return train_rows, val_rows


def _cycle_to(rows: list[dict], target: int) -> list[dict]:
    """Deterministically repeat/truncate ``rows`` to exactly ``target`` rows
    via round-robin cycling. No RNG, no new seed knob — #443's scope does
    not authorize one, and the downstream HF Trainer's default sampler
    already shuffles every epoch, so row ORDER out of this function is not
    load-bearing; only per-source row COUNT is.
    """
    if target <= len(rows):
        return rows[:target]
    out = list(rows)
    i = 0
    while len(out) < target:
        out.append(rows[i % len(rows)])
        i += 1
    return out


def _apportion(probs: tuple[float, ...], total: int) -> list[int]:
    """Largest-remainder apportionment: integer per-dataset quotas summing
    to exactly ``total``, as close as possible to ``probs[i] * total``.
    Deterministic; ties break by dataset index.
    """
    raw = [p * total for p in probs]
    base = [int(x) for x in raw]
    remainder = total - sum(base)
    order = sorted(range(len(probs)), key=lambda i: (raw[i] - base[i]), reverse=True)
    for i in order[:remainder]:
        base[i] += 1
    return base


def _combine_interleaved(per_dataset_rows: list[list[dict]], spec) -> list[dict]:
    """Combine per-dataset row lists per an InterleaveSpec (#443).

    ``concat`` / ``under`` / ``over`` / ``probs`` control per-source row
    COUNT, not order — see _cycle_to's docstring for why order doesn't
    matter here.
    """
    sizes = [len(rows) for rows in per_dataset_rows]
    if any(n == 0 for n in sizes):
        raise ValueError(
            "data.interleave: every dataset in data.train must have "
            ">= 1 row after formatting"
        )
    if spec.strategy == "concat":
        combined: list[dict] = []
        for rows in per_dataset_rows:
            combined.extend(rows)
        return combined
    if spec.strategy == "under":
        target = min(sizes)
        combined = []
        for rows in per_dataset_rows:
            combined.extend(rows[:target])
        return combined
    if spec.strategy == "over":
        target = max(sizes)
        combined = []
        for rows in per_dataset_rows:
            combined.extend(_cycle_to(rows, target))
        return combined
    if spec.strategy == "probs":
        quotas = _apportion(spec.probs, sum(sizes))
        combined = []
        for rows, quota in zip(per_dataset_rows, quotas):
            combined.extend(_cycle_to(rows, quota))
        return combined
    raise AssertionError(f"unreachable interleave strategy {spec.strategy!r}")


def _load_interleaved_local_datasets(
    train_paths: list[str],
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> dict:
    """#443: load + combine every local dataset in a list-shaped
    data.train per data.interleave, then hand the combined rows to
    _finalize(): the same seam every other loader uses. #680: "over"/
    "probs" now carve val out per source before cycling (_split_val),
    since those are the strategies that pad a source with copies of its
    own rows; "concat"/"under" are unchanged from #443.

    #459 — data_config.streaming=true (list may then also contain remote
    URIs; schema already enforced that) delegates to
    _load_interleaved_streaming_datasets instead: HF's own
    interleave_datasets/concatenate_datasets replace this function's
    eager per-file combining. The eager body below is untouched and only
    ever reached when streaming is false, so its output stays
    byte-identical to pre-#459 (pinned by
    test_single_path_train_output_is_byte_identical_to_baseline's sibling
    interleave goldens in tests/test_issue443_interleave_wiring.py).
    """
    from soup_cli.utils.data_pipeline import parse_interleave

    spec = parse_interleave(data_config.interleave, num_datasets=len(train_paths))
    if spec is None:
        # Defence-in-depth only: SoupConfig._validate_interleave_compat
        # already requires data.interleave whenever data.train is a list,
        # so a caller only reaches here via a hand-built DataConfig that
        # skipped SoupConfig validation.
        raise ValueError("data.train is a list but data.interleave is not set")

    if data_config.streaming:
        return _load_interleaved_streaming_datasets(
            train_paths,
            data_config,
            spec,
            preserve_source_columns=preserve_source_columns,
        )

    per_dataset_rows = [
        _load_one_local_dataset(
            p,
            data_config,
            preserve_source_columns=preserve_source_columns,
        )
        for p in train_paths
    ]

    if data_config.val_split > 0 and spec.strategy in ("over", "probs"):
        # #680: these two strategies pad a source with copies of its own
        # rows, so splitting the padded result can put one copy in train
        # and another in val. Carve val out of each source first instead.
        per_dataset_train = []
        per_dataset_val = []
        for rows in per_dataset_rows:
            train_rows, val_rows = _split_val_per_source(
                rows, data_config.val_split, spec.strategy
            )
            per_dataset_train.append(train_rows)
            per_dataset_val.append(val_rows)
        combined_train = _combine_interleaved(per_dataset_train, spec)
        combined_val = [row for val_rows in per_dataset_val for row in val_rows]
        console.print(
            f"[dim]Interleaved {len(train_paths)} datasets "
            f"(strategy={spec.strategy}) -> {len(combined_train)} train, "
            f"{len(combined_val)} val rows[/]"
        )
        return _finalize(
            combined_train,
            data_config,
            val=combined_val,
            preserve_source_columns=preserve_source_columns,
        )

    combined = _combine_interleaved(per_dataset_rows, spec)
    console.print(
        f"[dim]Interleaved {len(train_paths)} datasets "
        f"(strategy={spec.strategy}) -> {len(combined)} rows[/]"
    )
    return _finalize(
        combined,
        data_config,
        preserve_source_columns=preserve_source_columns,
    )


def _load_interleaved_streaming_datasets(
    train_paths: list[str],
    data_config: DataConfig,
    spec,
    *,
    preserve_source_columns: bool = False,
) -> dict:
    """#459 — data.interleave + data.streaming=true: delegate combining to
    HF ``datasets.interleave_datasets`` / ``concatenate_datasets`` instead
    of reimplementing mixing over an unbounded/remote source we can't
    count ahead of time.

    Every entry (local file path or remote URI — schema guarantees no hub
    names reach here) becomes one streaming dataset via
    ``datasets.load_dataset(builder, data_files=entry, split="train",
    streaming=True)``, where ``builder`` is chosen per entry from its file
    suffix (:func:`_streaming_builder_for`) — the same
    ``SUPPORTED_EXTENSIONS`` set the non-streaming local path
    (:func:`load_raw_data`) dispatches on, so flipping only
    ``data.streaming: true`` doesn't silently reinterpret a csv/txt/parquet
    file as JSON. Suffix-based selection works identically for local paths
    and remote URIs, which is why the two can share one streaming list.

    Strategy -> HF call mapping (the "mean the same thing as the local
    path" decision #459 asks for; also documented in docs/data.md):

    - ``concat`` -> ``concatenate_datasets(streams)``: same as the local
      path's "extend every source's rows in order", exactly.
    - ``under``  -> ``interleave_datasets(streams,
      stopping_strategy="first_exhausted")``: local ``under`` truncates
      every source to ``min(sizes)``; streaming can't know sizes ahead of
      time, so stopping at the first exhausted stream gives the same
      *shape* of outcome (bounded by the smallest source) without an exact
      row-count guarantee.
    - ``over``   -> ``interleave_datasets(streams,
      stopping_strategy="all_exhausted")``: local ``over`` upsamples every
      source to ``max(sizes)`` by cycling; "all exhausted" recycles
      shorter streams until the longest is consumed once — same shape,
      not exact count.
    - ``probs``  -> ``interleave_datasets(streams,
      probabilities=list(spec.probs), stopping_strategy="first_exhausted")``:
      local ``probs`` apportions an EXACT ratio via largest-remainder;
      streaming samples per ``probabilities`` instead, which converges to
      the same ratio asymptotically rather than exactly. Tested by running
      one probs config through both paths and comparing proportions
      (not two tests that each pass alone) — see
      test_issue459_interleave_streaming_hub.py.

    Combined output is eagerly materialised up to MAX_REMOTE_ROWS (shared
    with _load_remote_dataset — one OOM ceiling, not two). Format
    detection runs ONCE on the combined raw rows when format="auto",
    mirroring _load_remote_dataset's existing single-detect behaviour —
    unlike the local-file interleave path's per-source auto-detect,
    streaming sources are assumed schema-homogeneous; mixing differently
    -formatted streams via streaming interleave is out of scope (use the
    non-streaming local path for that).

    Security: every entry classified as a remote URI is canonicalised
    through ``validate_remote_uri`` before it reaches ``hf_load`` — the
    same allowlist validator _load_remote_dataset already runs for a
    single URI (bucket regex, no userinfo / query / fragment; a query
    string is SSRF-adjacent since fsspec backends treat it as a config
    override). The schema-time classifier only checks for a ``"://"``
    (scheme-agnostic — #468 review fix), so this call is the one place
    that actually validates the URI shape AND enforces the scheme
    allowlist — without it, a crafted entry like
    ``s3://bucket/key.jsonl?endpoint=http://169.254.169.254`` would reach
    hf_load unvalidated, and a non-allowlisted-scheme entry like
    ``https://example.com/data.jsonl`` would reach it unvalidated too.

    Media paths (vision / audio formats) resolve per entry, as on the eager
    path: against ``data.image_dir`` / ``data.audio_dir`` when set, else the
    entry's own directory (a remote entry has none). When the entries do not
    share that directory, each stream is tagged with its entry's index before
    the streams are combined -- lazily, through ``IterableDataset.map`` -- and
    the tag is popped as the rows materialise (:func:`_streams_need_source_tags`).
    """
    try:
        from datasets import concatenate_datasets, interleave_datasets
        from datasets import load_dataset as hf_load
    except ImportError as exc:
        raise ImportError(
            "data.streaming=true requires the 'datasets' package: "
            "pip install datasets"
        ) from exc

    from soup_cli.utils.data_pipeline import validate_remote_uri

    streams = []
    for entry in train_paths:
        load_entry = validate_remote_uri(entry) if _looks_like_remote_uri(entry) else entry
        builder = _streaming_builder_for(load_entry)
        ds = hf_load(builder, data_files=load_entry, split="train", streaming=True)
        buf = data_config.buffer_size
        if buf:
            ds = ds.shuffle(buffer_size=buf)
        streams.append(ds)

    tag_sources = _streams_need_source_tags(streams, data_config, train_paths)
    if tag_sources:
        streams = _tag_streams(streams)

    if spec.strategy == "concat":
        combined_stream = concatenate_datasets(streams)
    elif spec.strategy == "under":
        combined_stream = interleave_datasets(streams, stopping_strategy="first_exhausted")
    elif spec.strategy == "over":
        combined_stream = interleave_datasets(streams, stopping_strategy="all_exhausted")
    elif spec.strategy == "probs":
        combined_stream = interleave_datasets(
            streams, probabilities=list(spec.probs), stopping_strategy="first_exhausted"
        )
    else:
        raise AssertionError(f"unreachable interleave strategy {spec.strategy!r}")

    raw_data: list[dict] = []
    row_sources: list[int] = []
    for i, row in enumerate(combined_stream):
        if i >= MAX_REMOTE_ROWS:
            console.print(
                f"[yellow]Interleaved streaming dataset truncated at "
                f"{MAX_REMOTE_ROWS:,} rows (use non-streaming local files "
                "for larger jobs).[/]"
            )
            break
        raw_row = dict(row)
        if tag_sources:
            row_sources.append(raw_row.pop(_SOURCE_INDEX_KEY))
        raw_data.append(raw_row)

    fmt = data_config.format
    if fmt == "auto":
        fmt = detect_format(raw_data)
        console.print(f"[dim]Auto-detected format: {fmt}[/]")

    kept: list[int] = []
    formatted = _format_rows(
        raw_data,
        fmt,
        preserve_source_columns=preserve_source_columns,
        kept_indices=kept,
    )
    formatted = _resolve_streamed_media(
        formatted,
        [row_sources[position] for position in kept] if tag_sources else None,
        fmt,
        data_config,
        train_paths,
    )

    console.print(
        f"[dim]Streaming-interleaved {len(train_paths)} datasets "
        f"(strategy={spec.strategy}) -> {len(formatted)} rows[/]"
    )
    # `over` recycles the shorter stream, so the materialised rows contain
    # duplicates and the positional split in `_finalize` could put a row and
    # its copy on opposite sides (#702). The other three strategies never
    # duplicate a row here, so they keep the ordinary path.
    if spec.strategy == "over" and data_config.val_split > 0:
        train_rows, val_rows = _split_val_deduplicated(
            formatted, data_config.val_split
        )
        return _finalize(
            train_rows,
            data_config,
            val=val_rows,
            preserve_source_columns=preserve_source_columns,
        )

    return _finalize(
        formatted,
        data_config,
        preserve_source_columns=preserve_source_columns,
    )


def _validate_vision_images(data: list[dict], image_dir: Path) -> list[dict]:
    """Validate and resolve image paths in vision dataset rows.

    Each row must have an 'image' key with a filename or path, a decoded PIL
    image, or a datasets Image struct ({'bytes': ..., 'path': ...}). Resolves
    relative paths against image_dir and rejects path traversal — a crafted
    llava/sharegpt4v row like ``{"image": "/etc/passwd"}`` must not be handed
    to ``PIL.Image.open``. Mirrors :func:`_validate_audio_files` (the sibling
    audio path got this fix in v0.71.32; the vision path was missed).
    """
    return _resolve_media_column(data, "image", image_dir)


def _validate_audio_files(data: list[dict], audio_dir: Path) -> list[dict]:
    """Validate and resolve audio file paths in audio dataset rows.

    Each row must have an 'audio' key with a filename or path.
    Resolves relative paths against audio_dir. Rejects path traversal.
    """
    return _resolve_media_column(data, "audio", audio_dir)


# Row key of a media format -> the setting that names its base directory.
_MEDIA_DIR_SETTINGS = {"image": "data.image_dir", "audio": "data.audio_dir"}

# Set on streamed rows by _tag_source_index and popped as the rows
# materialise: the data.train index the row was read from.
_SOURCE_INDEX_KEY = "__soup_source_index__"


def _media_key(fmt: str) -> str | None:
    """The row key that holds a media path in ``fmt``; None for a text format."""
    if is_vision_format(fmt):
        return "image"
    if is_audio_format(fmt):
        return "audio"
    return None


def _media_dir_for(key: str, data_config: DataConfig, data_dir: Path | None) -> Path | None:
    """``data.image_dir`` / ``data.audio_dir`` when set, else ``data_dir``."""
    configured = data_config.image_dir if key == "image" else data_config.audio_dir
    return Path(configured) if configured else data_dir


def _entry_media_dir(key: str, data_config: DataConfig, entry: str) -> Path | None:
    """The base directory of one data.train entry's media; None for a remote
    entry when no directory is configured."""
    data_dir = None if _looks_like_remote_uri(entry) else Path(entry).parent
    return _media_dir_for(key, data_config, data_dir)


def _media_dir_required(source: str, key: str, *, plural: bool = False) -> ValueError:
    """The error for media paths that have no local directory to resolve against."""
    setting = _MEDIA_DIR_SETTINGS[key]
    if plural:
        subject = f"{source} are not local files, so their"
    else:
        subject = f"{source} is not a local file, so its"
    return ValueError(
        f"{subject} {key!r} paths have no directory to resolve against. "
        f"Set {setting} to the local directory that holds the {key} files: "
        f"relative paths resolve under it, and rows whose {key} is outside it "
        "are skipped."
    )


def _resolve_media_entries(
    rows: list[dict],
    key: str,
    media_dir: Path | None,
    *,
    source: str = "data.train",
    plural: bool = False,
) -> tuple[list[dict | None], int, int]:
    """Resolve each row's ``key`` media path against ``media_dir``.

    Returns one entry per row, in order -- the row with its path resolved,
    the row unchanged when its value is not a path at all (an image a Hub
    dataset has already decoded), or None for a skipped row -- plus how many
    rows were skipped for a missing path and for a path outside
    ``media_dir``. Every load path goes through here, so a row is treated
    the same way wherever it was read from.

    A relative path resolves against ``media_dir``. A network-share or
    device form (:func:`~soup_cli.utils.paths.is_network_or_device_path`), a
    value holding a NUL byte, and a ``bytes`` or path-object value count as
    outside and are refused before any filesystem call. ``media_dir`` is
    None for a remote URI or Hub dataset with no ``data.image_dir`` /
    ``data.audio_dir``: a string path there has nothing to resolve against,
    so the load stops and names the setting (``source``, ``plural``).
    """
    from soup_cli.utils.paths import is_network_or_device_path, is_under

    entries: list[dict | None] = []
    missing = 0
    outside = 0
    for row in rows:
        value = row.get(key)
        if not value:
            missing += 1
            entries.append(None)
            continue
        if isinstance(value, (bytes, os.PathLike)):
            # A reader takes these as a file name too, but they are not path
            # strings: skipped like a path outside, never handed on unchecked.
            outside += 1
            entries.append(None)
            continue
        if not isinstance(value, str):
            if isinstance(value, dict):
                # Struct with embedded bytes passes through unchanged
                if value.get("bytes"):
                    entries.append(row)
                    continue
                path_val = value.get("path")
                if path_val:
                    if isinstance(path_val, (bytes, os.PathLike)):
                        outside += 1
                        entries.append(None)
                        continue
                    if not isinstance(path_val, str):
                        missing += 1
                        entries.append(None)
                        continue
                    if media_dir is None:
                        raise _media_dir_required(source, key, plural=plural)
                    if "\x00" in path_val or is_network_or_device_path(path_val):
                        outside += 1
                        entries.append(None)
                        continue
                    media_path = Path(path_val)
                    if not media_path.is_absolute():
                        media_path = media_dir / media_path
                    if not is_under(media_path, media_dir):
                        outside += 1
                        entries.append(None)
                        continue
                    entries.append({**row, key: {**value, "path": str(media_path.resolve())}})
                    continue
            entries.append(row)
            continue
        if media_dir is None:
            raise _media_dir_required(source, key, plural=plural)
        if "\x00" in value or is_network_or_device_path(value):
            outside += 1
            entries.append(None)
            continue
        media_path = Path(value)
        if not media_path.is_absolute():
            media_path = media_dir / media_path
        # realpath + commonpath (is_under) -- Path.is_relative_to() breaks on
        # Windows 8.3 short names.
        if not is_under(media_path, media_dir):
            outside += 1
            entries.append(None)
            continue
        entries.append({**row, key: str(media_path.resolve())})
    return entries, missing, outside


def _warn_media_skips(key: str, missing: int, outside: int, media_dir: Path | None) -> None:
    """The "rows skipped" warnings of one media pass, worded as they always were.

    ``media_dir`` is None only for rows with no local directory, where the
    one value counted as outside without a string path is a ``bytes`` or
    path object.
    """
    if missing > 0:
        console.print(f"[yellow]Warning: {missing} rows skipped (missing {key} path)[/]")
    if outside > 0 and key == "image":
        if media_dir is None:
            reason = "image value is not a path string"
        else:
            reason = f"image path outside {media_dir}"
        console.print(f"[yellow]Warning: {outside} rows skipped ({reason})[/]")
    elif outside > 0:
        console.print(
            f"[red]Warning: {outside} rows skipped (audio path traversal blocked)[/]"
        )


def _resolve_media_column(
    rows: list[dict],
    key: str,
    media_dir: Path | None,
    *,
    source: str = "data.train",
    plural: bool = False,
) -> list[dict]:
    """:func:`_resolve_media_entries` plus its warnings; returns the rows kept."""
    entries, missing, outside = _resolve_media_entries(
        rows, key, media_dir, source=source, plural=plural
    )
    _warn_media_skips(key, missing, outside, media_dir)
    return [row for row in entries if row is not None]


def _resolve_remote_media(
    rows: list[dict], fmt: str, data_config: DataConfig, *, source: str
) -> list[dict]:
    """Media pass for rows that did not come from a local file.

    A remote URI or a Hub dataset has no local directory to stand in for the
    data file's, so a string media path resolves only against
    ``data.image_dir`` / ``data.audio_dir``. Rows of a text format are
    returned unchanged.
    """
    key = _media_key(fmt)
    if key is None:
        return rows
    return _resolve_media_column(rows, key, _media_dir_for(key, data_config, None), source=source)


def _streams_need_source_tags(
    streams: list, data_config: DataConfig, train_paths: list[str]
) -> bool:
    """Whether streamed rows must carry the index of the entry they came from.

    Only a media row needs it, and only when the entries do not share one base
    directory for its key: otherwise every row resolves against that base.
    ``format: auto`` is resolved after the rows materialise, from the first
    rows of the combined stream, which are among the first rows of each
    stream; the format found can only be a media format if one of those rows
    carries its key, so the streams are tagged when one does.
    """
    keys = [
        key
        for key in _MEDIA_DIR_SETTINGS
        if len({_entry_media_dir(key, data_config, entry) for entry in train_paths}) > 1
    ]
    if not keys:
        return False
    if data_config.format != "auto":
        return _media_key(data_config.format) in keys
    return any(
        key in row
        for stream in streams
        for row in islice(stream, MAX_DETECT_ROWS)
        for key in keys
    )


def _tag_source_index(row: dict, index: int, columns: tuple[str, ...] = ()) -> dict:
    """``IterableDataset.map`` callback: the data.train index of ``row``, plus
    None for each of ``columns`` the row lacks."""
    tag: dict = {name: None for name in columns if name not in row}
    tag[_SOURCE_INDEX_KEY] = index
    return tag


def _tag_streams(streams: list) -> list:
    """Tag every stream's rows with its data.train index.

    A stream whose features are known keeps them, plus the int64 tag, so
    ``datasets`` does not re-infer its types from the first rows. A row gets
    each column another stream declares and it lacks, as None: combining
    untagged streams fills those in the same way.
    """
    columns = tuple(
        dict.fromkeys(
            name for stream in streams if stream.features is not None for name in stream.features
        )
    )
    tagged = []
    for index, stream in enumerate(streams):
        tag = partial(_tag_source_index, index=index, columns=columns)
        if stream.features is None:
            tagged.append(stream.map(tag))
            continue
        from datasets import Features, Value

        features = Features({**stream.features, _SOURCE_INDEX_KEY: Value("int64")})
        tagged.append(stream.map(tag, features=features))
    return tagged


def _resolve_streamed_media(
    rows: list[dict],
    row_sources: list[int] | None,
    fmt: str,
    data_config: DataConfig,
    train_paths: list[str],
) -> list[dict]:
    """Media pass of the streaming interleave path: each row against its own entry.

    ``row_sources[i]`` is the data.train index ``rows[i]`` was streamed from.
    A local entry's default directory is its file's parent, as on the eager
    path; a remote entry has none. Each entry is resolved and warned about
    on its own, like the eager path's per-file warnings, and the rows keep
    their stream order. ``row_sources`` is None when the streams were not
    tagged: every entry then has the same base for this format's key.
    """
    key = _media_key(fmt)
    if key is None:
        return rows
    if row_sources is None:
        names = ", ".join(repr(entry) for entry in train_paths)
        return _resolve_media_column(
            rows,
            key,
            _entry_media_dir(key, data_config, train_paths[0]),
            source=f"data.train entries {names}",
            plural=True,
        )
    positions: dict[int, list[int]] = {}
    for position, source_index in enumerate(row_sources):
        positions.setdefault(source_index, []).append(position)
    resolved: list[dict | None] = list(rows)
    for source_index in sorted(positions):
        entry = train_paths[source_index]
        media_dir = _entry_media_dir(key, data_config, entry)
        group = positions[source_index]
        entries, missing, outside = _resolve_media_entries(
            [rows[position] for position in group],
            key,
            media_dir,
            source=f"data.train[{source_index}] {entry!r}",
        )
        _warn_media_skips(key, missing, outside, media_dir)
        for position, entry_row in zip(group, entries):
            resolved[position] = entry_row
    return [row for row in resolved if row is not None]


def _looks_like_remote_uri(value: str) -> bool:
    """Quick sniff for a URI-shaped data.train entry (contains '://').

    Deliberately scheme-agnostic (#468 review fix) — classification and
    dispatch only need to know "does this want to be treated as a remote
    source", not "is the scheme one we allow". Scheme-allowlist
    enforcement happens once, downstream, in validate_remote_uri, which
    refuses a non-allowlisted scheme BY NAME — narrowing this sniff to
    allowlisted schemes only (as it did before this fix, via is_remote_uri)
    let https/http/ftp (with a familiar file suffix) slip into 'local'
    classification and reach hf_load completely unvalidated, bypassing
    validate_remote_uri rather than being refused by it.
    """
    return isinstance(value, str) and "://" in value


def _load_remote_dataset(
    train_path: str,
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> dict:
    """Load JSONL from a remote fsspec URI (s3 / gs / az / oci / etc.).

    Validates the URI via the v0.42.0 ``validate_remote_uri`` allowlist
    (bucket regex, no userinfo/query/fragment) BEFORE opening any
    connection — defends against URL injection into the fsspec backend.

    Streaming knobs (``data_config.streaming`` + ``buffer_size`` + ``shards``)
    are honoured via :func:`datasets.load_dataset` when present; otherwise
    the file is streamed as JSONL through :func:`fsspec.open`.

    Image / audio paths resolve against ``data.image_dir`` /
    ``data.audio_dir`` (:func:`_resolve_remote_media`): a remote object has
    no local directory of its own.
    """
    from soup_cli.utils.data_pipeline import (
        required_remote_package,
        validate_remote_uri,
    )

    canonical = validate_remote_uri(train_path)
    scheme = canonical.split("://", 1)[0]

    try:
        import fsspec  # type: ignore[import-not-found]
    except ImportError:
        from rich.panel import Panel

        pkg = required_remote_package(scheme) or scheme
        console.print(
            Panel(
                f"[bold yellow]Missing dependency:[/] reading from "
                f"[bold]{scheme}://[/] requires the [bold]{pkg}[/] package.\n\n"
                f"Install with:\n  [bold]pip install {pkg}[/]",
                title="Remote loader",
                border_style="yellow",
            )
        )
        raise

    # Try the HF datasets streaming path first when the user opted in via
    # ``data.streaming=true`` — gives us free interleaving, shuffling, and
    # caching. Falls back to direct fsspec.open when datasets is missing or
    # rejects the URI.
    if data_config.streaming:
        try:
            from datasets import load_dataset as hf_load
        except ImportError as exc:
            raise ImportError(
                "data.streaming=true requires the 'datasets' package: "
                "pip install datasets"
            ) from exc
        ds = hf_load(
            "json",
            data_files=canonical,
            split="train",
            streaming=True,
        )
        buf = data_config.buffer_size
        if buf:
            ds = ds.shuffle(buffer_size=buf)
        # Eager materialise capped at MAX_REMOTE_ROWS — emit a clear advisory
        # if the cap trips.
        raw_data: list[dict] = []
        for i, row in enumerate(ds):
            if i >= MAX_REMOTE_ROWS:
                console.print(
                    f"[yellow]Remote dataset truncated at {MAX_REMOTE_ROWS:,} "
                    f"rows (use a local split for larger jobs).[/]"
                )
                break
            raw_data.append(row)
    else:
        # Non-streaming: open once, read lines, decode JSON.
        raw_data = []
        with fsspec.open(canonical, mode="rt", encoding="utf-8-sig") as fh:
            for i, raw_line in enumerate(fh):
                if i >= MAX_REMOTE_ROWS:
                    console.print(
                        f"[yellow]Remote dataset truncated at "
                        f"{MAX_REMOTE_ROWS:,} rows.[/]"
                    )
                    break
                stripped = raw_line.strip()
                if not stripped:
                    continue
                try:
                    raw_data.append(json.loads(stripped))
                except json.JSONDecodeError as exc:
                    console.print(
                        f"[yellow]Warning: invalid JSON on line "
                        f"{i + 1}: {exc}[/]"
                    )

    fmt = data_config.format
    if fmt == "auto":
        fmt = detect_format(raw_data)
        console.print(f"[dim]Auto-detected format: {fmt}[/]")

    formatted = _format_rows(
        raw_data,
        fmt,
        preserve_source_columns=preserve_source_columns,
    )
    formatted = _resolve_remote_media(
        formatted, fmt, data_config, source=f"Remote dataset {canonical!r}"
    )

    return _finalize(
        formatted,
        data_config,
        preserve_source_columns=preserve_source_columns,
    )


def _rows_from_hub_split(split, data_config: DataConfig, *, shuffle: bool) -> list[dict]:
    """Materialise one hub split.

    Streaming matches `_load_remote_dataset`: optional shuffle buffer, then
    eager rows capped at MAX_REMOTE_ROWS. Non-streaming is the pre-#689
    list comprehension (no cap).
    """
    if data_config.streaming:
        buf = data_config.buffer_size
        if shuffle and buf:
            split = split.shuffle(buffer_size=buf)
        raw_data: list[dict] = []
        for i, row in enumerate(split):
            if i >= MAX_REMOTE_ROWS:
                console.print(
                    f"[yellow]Hub dataset truncated at {MAX_REMOTE_ROWS:,} "
                    f"rows (use a local split for larger jobs).[/]"
                )
                break
            raw_data.append(dict(row))
        return raw_data
    return [dict(row) for row in split]


def _load_one_hub_dataset(
    name: str,
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> tuple[list[dict], list[dict] | None]:
    """Load + format one HF-hub dataset name's 'train' (+ optional
    'validation') split. Factored out of _load_hf_dataset (#459) — same
    pattern #443 used for _load_one_local_dataset — so the hub interleave
    path below reuses this rather than re-deriving it. _load_hf_dataset's
    own single-name behaviour is left byte-identical by construction: it
    now just calls this and forwards straight to _finalize, same as before.

    Image / audio paths in both splits resolve against ``data.image_dir`` /
    ``data.audio_dir`` (:func:`_resolve_remote_media`): a Hub dataset has
    no local directory of its own.
    """
    try:
        from datasets import load_dataset as hf_load
    except ImportError:
        raise ImportError("Install datasets: pip install datasets")

    console.print(f"[dim]Loading from HuggingFace: {name}[/]")
    if data_config.streaming:
        ds = hf_load(name, streaming=True)
    else:
        ds = hf_load(name)

    if "train" not in ds:
        raise ValueError(f"Dataset {name} has no 'train' split")

    raw_data = _rows_from_hub_split(ds["train"], data_config, shuffle=True)
    fmt = data_config.format
    if fmt == "auto":
        fmt = detect_format(raw_data)

    formatted = _format_rows(
        raw_data,
        fmt,
        preserve_source_columns=preserve_source_columns,
    )
    formatted = _resolve_remote_media(
        formatted, fmt, data_config, source=f"Hub dataset {name!r}"
    )

    if "validation" in ds:
        val_data = _rows_from_hub_split(
            ds["validation"], data_config, shuffle=False,
        )
        val_formatted = _format_rows(
            val_data,
            fmt,
            preserve_source_columns=preserve_source_columns,
        )
        val_formatted = _resolve_remote_media(
            val_formatted, fmt, data_config, source=f"Hub dataset {name!r}"
        )
        return formatted, val_formatted

    return formatted, None


def _load_hf_dataset(
    name: str,
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> dict:
    """Load a dataset from HuggingFace Hub.

    The hub split wins over val_split; passing it through as _finalize's
    ``val=`` means _finalize does not re-derive one from the train rows.
    """
    formatted, val_formatted = _load_one_hub_dataset(
        name,
        data_config,
        preserve_source_columns=preserve_source_columns,
    )
    if val_formatted is not None:
        return _finalize(
            formatted,
            data_config,
            val=val_formatted,
            preserve_source_columns=preserve_source_columns,
        )
    return _finalize(
        formatted,
        data_config,
        preserve_source_columns=preserve_source_columns,
    )


def _load_interleaved_hub_datasets(
    train_names: list[str],
    data_config: DataConfig,
    *,
    preserve_source_columns: bool = False,
) -> dict:
    """#459 — data.train is a list of HF-hub dataset names combined via
    data.interleave. Reuses the SAME _combine_interleaved the local-file
    path (#443) uses — this is what makes the strategy names mean the same
    thing across both paths, by construction rather than by parallel
    reimplementation.

    Decided validation-split precedence (documented here + docs/data.md,
    per the issue's demand that this not be emergent): a hub entry's own
    'validation' split is honoured for the COMBINED result only when EVERY
    entry provides one: those are combined with the same spec and passed
    through as _finalize's val=. If only some entries provide one, it is
    ignored (warned, naming which entries had it) and data_config.val_split
    is derived instead, exactly as if no entry had a hub split. A
    partial-hub-split mixture would otherwise silently be a
    smaller/differently-composed val set than a reader expects. If no
    entry provides one, behaviour is unchanged from today. #680: that
    val_split fallback carves val out per source before over/probs pads
    it, same fix and same reason as _load_interleaved_local_datasets.
    """
    from soup_cli.utils.data_pipeline import parse_interleave

    spec = parse_interleave(data_config.interleave, num_datasets=len(train_names))
    if spec is None:
        # Defence-in-depth only — see _load_interleaved_local_datasets's
        # identical comment.
        raise ValueError("data.train is a list but data.interleave is not set")

    per_dataset_train: list[list[dict]] = []
    per_dataset_val: list[list[dict] | None] = []
    for name in train_names:
        train_rows, val_rows = _load_one_hub_dataset(
            name,
            data_config,
            preserve_source_columns=preserve_source_columns,
        )
        per_dataset_train.append(train_rows)
        per_dataset_val.append(val_rows)

    has_val = [v is not None for v in per_dataset_val]
    if all(has_val):
        combined_train = _combine_interleaved(per_dataset_train, spec)
        combined_val = _combine_interleaved(
            [v for v in per_dataset_val if v is not None], spec
        )
        result = _finalize(
            combined_train,
            data_config,
            val=combined_val,
            preserve_source_columns=preserve_source_columns,
        )
    else:
        if any(has_val):
            missing = [
                name for name, v in zip(train_names, per_dataset_val) if v is None
            ]
            if spec.strategy in ("over", "probs"):
                detail = "carving data.val_split out per source before oversampling"
            else:
                detail = "applying data.val_split to the combined train rows instead"
            console.print(
                "[yellow]data.interleave: only some HF-hub datasets provide "
                f"a 'validation' split (missing from {missing}) — ignoring "
                f"every hub validation split and {detail}.[/]"
            )
        if data_config.val_split > 0 and spec.strategy in ("over", "probs"):
            per_split = [
                _split_val_per_source(rows, data_config.val_split, spec.strategy)
                for rows in per_dataset_train
            ]
            combined_train = _combine_interleaved([t for t, _ in per_split], spec)
            combined_val = [row for _, v in per_split for row in v]
            result = _finalize(
                combined_train,
                data_config,
                val=combined_val,
                preserve_source_columns=preserve_source_columns,
            )
        else:
            combined_train = _combine_interleaved(per_dataset_train, spec)
            result = _finalize(
                combined_train,
                data_config,
                preserve_source_columns=preserve_source_columns,
            )

    console.print(
        f"[dim]Interleaved {len(train_names)} HF-hub datasets "
        f"(strategy={spec.strategy}) -> {len(combined_train)} rows[/]"
    )
    return result