Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

32.5 KB
Raw Blame
"""`data.interleave` extended to `data.streaming: true` and HF-hub dataset
name lists (issue #459 — the follow-up #443 explicitly deferred).

#443 wired `data.interleave` for a `data.train` list of local file paths
only; its own schema validator refused `data.streaming: true` and any
remote-URI / HF-hub-name list entry outright, naming this issue as the
follow-up. #459 lifts that refusal for two DECIDED (not emergent) new
shapes, each dispatched by classifying every `data.train` entry as
`local` / `remote` / `hub` (`loader._classify_train_entry`,
`schema._validate_interleave_compat`'s inline `_kind`):

- All entries `local` / `remote`, `data.streaming: true` — delegates to
  `datasets.interleave_datasets` / `concatenate_datasets`
  (`loader._load_interleaved_streaming_datasets`). The strategy-name
  mapping is documented in that function's docstring and in
  `docs/data.md`; the acceptance criterion is that the strategies MEAN THE
  SAME THING as the local path, verified below by running one `probs`
  config through both paths and comparing the resulting proportions —
  not two tests that each pass in isolation.
- All entries `hub` (HF-hub dataset names) — eager per-entry loading via
  the existing hub loader, reusing the SAME `_combine_interleaved` the
  local path uses (`loader._load_interleaved_hub_datasets`). The
  `val_split`-vs-hub-split precedence is decided here: a combined
  validation split is honoured only when EVERY entry provides one;
  otherwise it's ignored (warned) and `val_split` applies to the combined
  train rows.
- Anything still unsupported (a streaming hub-name list; a list mixing
  hub names with local/remote entries) keeps refusing, by name — covered
  by tests/test_issue443_interleave_wiring.py's updated refusal tests.
"""

from __future__ import annotations

import json
import random
import sys
from pathlib import Path

import pytest
from pydantic import ValidationError

from soup_cli.config.schema import DataConfig, SoupConfig
from soup_cli.data.loader import load_dataset


def _write_jsonl(path: Path, texts: list[str]) -> None:
    path.write_text(
        "\n".join(f'{{"text": {t!r}}}' for t in texts).replace("'", '"') + "\n",
        encoding="utf-8",
    )


def _cfg(tmp_path: Path, **data_overrides) -> SoupConfig:
    data = {
        "train": str(tmp_path / "a.jsonl"),
        "format": "plaintext",
        "val_split": 0.0,
    }
    data.update(data_overrides)
    return SoupConfig.model_validate(
        {
            "base": "test-base",
            "task": "sft",
            "data": data,
            "training": {"epochs": 1},
            "output": str(tmp_path / "out"),
        }
    )


# ---------------------------------------------------------------------------
# Fake `datasets` module — no network, no real HF dependency.
# ---------------------------------------------------------------------------


class _FakeIterableDataset:
    """Stand-in for HF's IterableDataset — just an iterable of dict rows."""

    def __init__(self, rows):
        self._rows = list(rows)

    def shuffle(self, buffer_size=None):
        return self  # order isn't load-bearing here — see loader's _cycle_to note

    def __iter__(self):
        return iter(self._rows)


def _rows_from_jsonl(path) -> list[dict]:
    text = Path(path).read_text(encoding="utf-8-sig")
    return [json.loads(line) for line in text.splitlines() if line.strip()]


def _round_robin(pools: list[list[dict]], stopping_strategy: str) -> list[dict]:
    """Realistic-enough fake for interleave_datasets(streams, stopping_strategy=...)
    with no probabilities: alternate one row per source per round.
    """
    if any(len(p) == 0 for p in pools):
        return []
    out: list[dict] = []
    if stopping_strategy == "first_exhausted":
        n = min(len(p) for p in pools)
    elif stopping_strategy == "all_exhausted":
        n = max(len(p) for p in pools)
    else:
        raise AssertionError(f"unexpected stopping_strategy {stopping_strategy!r}")
    for i in range(n):
        for pool in pools:
            out.append(pool[i % len(pool)])
    return out


def _weighted_sample(
    pools: list[list[dict]], probabilities: list[float], stopping_strategy: str
) -> list[dict]:
    """Realistic-enough fake for interleave_datasets(streams, probabilities=...):
    weighted-without-replacement draw per source, stopping per policy. With
    large-enough pools this converges to `probabilities` — that convergence
    is exactly what the acceptance test below checks for.
    """
    rng = random.Random(0)
    ptrs = [0] * len(pools)
    out: list[dict] = []
    max_draws = sum(len(p) for p in pools) * 10
    for _ in range(max_draws):
        i = rng.choices(range(len(pools)), weights=probabilities, k=1)[0]
        if ptrs[i] >= len(pools[i]):
            if stopping_strategy == "first_exhausted":
                break
            ptrs[i] = 0
        out.append(pools[i][ptrs[i]])
        ptrs[i] += 1
    return out


def _install_fake_streaming_datasets(monkeypatch, calls: list):
    """Installs a fake `datasets` module recording every
    interleave_datasets/concatenate_datasets call (for the call-mapping
    assertions) while still combining real rows (for the non-degenerate /
    proportions assertions)."""

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        assert builder == "json"
        assert split == "train"
        assert streaming is True
        return _FakeIterableDataset(_rows_from_jsonl(data_files))

    def fake_concatenate_datasets(streams):
        calls.append(("concatenate_datasets", {}))
        combined = []
        for s in streams:
            combined.extend(list(s))
        return _FakeIterableDataset(combined)

    def fake_interleave_datasets(
        streams, probabilities=None, stopping_strategy="first_exhausted"
    ):
        calls.append((
            "interleave_datasets",
            {"probabilities": probabilities, "stopping_strategy": stopping_strategy},
        ))
        pools = [list(s) for s in streams]
        if probabilities is not None:
            combined = _weighted_sample(pools, probabilities, stopping_strategy)
        else:
            combined = _round_robin(pools, stopping_strategy)
        return _FakeIterableDataset(combined)

    fake_module = type(sys)("datasets")
    fake_module.load_dataset = fake_load_dataset
    fake_module.concatenate_datasets = fake_concatenate_datasets
    fake_module.interleave_datasets = fake_interleave_datasets
    monkeypatch.setitem(sys.modules, "datasets", fake_module)


# ---------------------------------------------------------------------------
# Schema-level: the four dispatch outcomes from _validate_interleave_compat
# ---------------------------------------------------------------------------


def test_streaming_local_list_validates():
    cfg = DataConfig(
        train=["a.jsonl", "b.jsonl"], interleave="concat", format="plaintext", streaming=True
    )
    assert cfg.streaming is True


def test_streaming_remote_uri_list_validates():
    cfg = DataConfig(
        train=["s3://bucket/a.jsonl", "s3://bucket/b.jsonl"],
        interleave="under",
        format="plaintext",
        streaming=True,
    )
    assert cfg.streaming is True


def test_remote_uri_without_streaming_rejected(tmp_path):
    with pytest.raises(ValidationError, match="require data.streaming=true"):
        _cfg(
            tmp_path,
            train=[str(tmp_path / "a.jsonl"), "s3://bucket/b.jsonl"],
            interleave="concat",
        )


def test_https_entry_classified_remote_not_local_or_hub(tmp_path):
    # #468 review finding: a non-allowlisted scheme (https/http/ftp) with a
    # familiar suffix must still classify 'remote', not 'local' — a plain
    # non-streaming config with a familiar-suffix https:// entry doesn't
    # discriminate (a 'local'-classified entry validates fine too), so this
    # asserts the SAME refusal 'remote' + no-streaming already gets above,
    # proving https classifies remote rather than silently local/hub.
    with pytest.raises(ValidationError, match="require data.streaming=true"):
        _cfg(
            tmp_path,
            train=[str(tmp_path / "a.jsonl"), "https://example.com/b.jsonl"],
            interleave="concat",
        )


def test_all_hub_list_without_streaming_validates(tmp_path):
    cfg = _cfg(
        tmp_path,
        train=["org/dataset-a", "org/dataset-b"],
        interleave="concat",
        format="plaintext",
    )
    assert cfg.data.train == ["org/dataset-a", "org/dataset-b"]
    assert cfg.data.interleave == "concat"
    assert cfg.data.streaming is False


def test_dotted_hub_names_classified_as_hub_not_local(tmp_path):
    # #468 review finding: a hub name with a version number was
    # misclassified as 'local' because Path(...).suffix is non-empty for
    # e.g. "OpenHermes-2.5" (.5) / "dclm-baseline-1.0" (.0). These are the
    # owner's own repro names, and docs/data.md's own example.
    #
    # Discriminating on validates-vs-doesn't (rather than just checking
    # cfg.data.train round-trips) matters here: a plain concat/no-streaming
    # config validates either way kinds get classified (an all-'local' list
    # and an all-'hub' list are both valid, non-mixed shapes), so it
    # wouldn't catch a misclassification. Pairing with streaming=True does
    # discriminate: 'hub' + streaming=True refuses (mirrors
    # test_all_hub_list_streaming_still_rejected); 'local' + streaming=True
    # would validate instead, silently missing the bug.
    with pytest.raises(ValidationError, match="does not support data.streaming=true"):
        _cfg(
            tmp_path,
            train=["teknium/OpenHermes-2.5", "mlfoundations/dclm-baseline-1.0"],
            interleave="concat",
            streaming=True,
        )


def test_all_hub_list_streaming_still_rejected(tmp_path):
    with pytest.raises(ValidationError, match="does not support data.streaming=true"):
        _cfg(
            tmp_path,
            train=["org/dataset-a", "org/dataset-b"],
            interleave="concat",
            streaming=True,
        )


def test_mixed_kinds_rejected(tmp_path):
    with pytest.raises(ValidationError, match="not a mix"):
        _cfg(
            tmp_path,
            train=[str(tmp_path / "a.jsonl"), "org/dataset-a"],
            interleave="concat",
        )


# ---------------------------------------------------------------------------
# Loader-level, streaming: strategy -> HF call mapping + non-degenerate
# ---------------------------------------------------------------------------


def test_streaming_concat_calls_concatenate_datasets(tmp_path, monkeypatch):
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(4)])
    _write_jsonl(tmp_path / "b.jsonl", [f"B-{i}" for i in range(3)])
    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")],
        interleave="concat",
        streaming=True,
    )
    result = load_dataset(cfg.data)
    assert calls == [("concatenate_datasets", {})]
    texts = [row["text"] for row in result["train"]]
    assert len(texts) == 7
    assert any(t.startswith("A-") for t in texts)
    assert any(t.startswith("B-") for t in texts)


def test_streaming_under_calls_interleave_first_exhausted(tmp_path, monkeypatch):
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(10)])
    _write_jsonl(tmp_path / "b.jsonl", [f"B-{i}" for i in range(3)])
    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")],
        interleave="under",
        streaming=True,
    )
    result = load_dataset(cfg.data)
    assert calls == [
        ("interleave_datasets", {"probabilities": None, "stopping_strategy": "first_exhausted"})
    ]
    texts = [row["text"] for row in result["train"]]
    assert any(t.startswith("A-") for t in texts)
    assert any(t.startswith("B-") for t in texts)
    # bounded by the smaller source, same shape as the local "under" path
    assert len(texts) == 6


def test_streaming_over_calls_interleave_all_exhausted(tmp_path, monkeypatch):
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(10)])
    _write_jsonl(tmp_path / "b.jsonl", [f"B-{i}" for i in range(3)])
    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")],
        interleave="over",
        streaming=True,
    )
    result = load_dataset(cfg.data)
    assert calls == [
        ("interleave_datasets", {"probabilities": None, "stopping_strategy": "all_exhausted"})
    ]
    texts = [row["text"] for row in result["train"]]
    # upsampled to match the larger source, same shape as the local "over" path
    assert len(texts) == 20
    assert sum(t.startswith("A-") for t in texts) == 10
    assert sum(t.startswith("B-") for t in texts) == 10


def test_streaming_probs_calls_interleave_with_probabilities(tmp_path, monkeypatch):
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(30)])
    _write_jsonl(tmp_path / "b.jsonl", [f"B-{i}" for i in range(10)])
    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")],
        interleave={"strategy": "probs", "probs": [0.75, 0.25]},
        streaming=True,
    )
    result = load_dataset(cfg.data)
    assert calls == [(
        "interleave_datasets",
        {"probabilities": [0.75, 0.25], "stopping_strategy": "first_exhausted"},
    )]
    texts = [row["text"] for row in result["train"]]
    assert any(t.startswith("A-") for t in texts)
    assert any(t.startswith("B-") for t in texts)


def test_streaming_local_and_remote_uris_can_mix_one_list(tmp_path, monkeypatch):
    # loader._classify_train_entry allows local + remote to share the
    # streaming dispatch — both go through the same "json" builder.
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(4)])

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        assert builder == "json"
        if str(data_files).startswith("s3://"):
            return _FakeIterableDataset([{"text": f"B-{i}"} for i in range(3)])
        return _FakeIterableDataset(_rows_from_jsonl(data_files))

    monkeypatch.setattr(sys.modules["datasets"], "load_dataset", fake_load_dataset)

    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), "s3://bucket/b.jsonl"],
        interleave="concat",
        streaming=True,
    )
    result = load_dataset(cfg.data)
    texts = [row["text"] for row in result["train"]]
    assert len(texts) == 7
    assert any(t.startswith("A-") for t in texts)
    assert any(t.startswith("B-") for t in texts)


def test_streaming_remote_uri_is_canonicalized_before_hf_load(tmp_path, monkeypatch):
    # Security regression: _load_interleaved_streaming_datasets must run each
    # remote entry through validate_remote_uri (same allowlist
    # _load_remote_dataset already applies to a single URI) before it
    # reaches hf_load — not the raw, unvalidated entry. Scheme case-folding
    # is the cheapest observable proof the canonical string (not the input)
    # is what's dispatched.
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(4)])

    received: list = []

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        received.append(data_files)
        if str(data_files).lower().startswith("s3://"):
            return _FakeIterableDataset([{"text": f"B-{i}"} for i in range(3)])
        return _FakeIterableDataset(_rows_from_jsonl(data_files))

    monkeypatch.setattr(sys.modules["datasets"], "load_dataset", fake_load_dataset)

    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), "S3://bucket/b.jsonl"],
        interleave="concat",
        streaming=True,
    )
    load_dataset(cfg.data)
    assert "s3://bucket/b.jsonl" in received
    assert "S3://bucket/b.jsonl" not in received


def test_streaming_malicious_query_uri_rejected_before_reaching_hf_load(tmp_path, monkeypatch):
    # HIGH — SSRF allowlist bypass: the schema-time classifier
    # (_classify_train_entry / _kind) only checks for "://" (scheme-
    # agnostic), so a query string — SSRF-adjacent, since fsspec backends
    # treat query params as config overrides — sails through the parse-time
    # gate. validate_remote_uri must still catch it inside the streaming
    # loader, before any entry ever reaches hf_load.
    _install_fake_streaming_datasets(monkeypatch, [])
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(4)])

    reached: list = []

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        reached.append(data_files)
        return _FakeIterableDataset([{"text": "unused"}])

    monkeypatch.setattr(sys.modules["datasets"], "load_dataset", fake_load_dataset)

    cfg = _cfg(
        tmp_path,
        train=[
            "s3://bucket/key.jsonl?endpoint=http://169.254.169.254",
            str(tmp_path / "a.jsonl"),
        ],
        interleave="concat",
        streaming=True,
    )
    with pytest.raises(ValueError, match="query string"):
        load_dataset(cfg.data)
    assert reached == []


def test_streaming_https_entry_rejected_by_name(tmp_path, monkeypatch):
    # MEDIUM — non-allowlisted schemes bypassed validation entirely: https
    # isn't in _REMOTE_SCHEMES, so the OLD scheme-allowlist sniff classified
    # "https://.../b.jsonl" as 'local' (familiar suffix) instead of
    # 'remote', skipping validate_remote_uri and reaching hf_load raw. The
    # scheme-agnostic "://" classifier now routes it through
    # validate_remote_uri, which refuses the scheme by name.
    _install_fake_streaming_datasets(monkeypatch, [])
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(4)])

    reached: list = []

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        reached.append(data_files)
        return _FakeIterableDataset([{"text": "unused"}])

    monkeypatch.setattr(sys.modules["datasets"], "load_dataset", fake_load_dataset)

    cfg = _cfg(
        tmp_path,
        train=["https://example.com/b.jsonl", str(tmp_path / "a.jsonl")],
        interleave="concat",
        streaming=True,
    )
    with pytest.raises(ValueError, match="not in allowlist"):
        load_dataset(cfg.data)
    assert reached == []


def test_streaming_legit_gcs_entry_still_passes(tmp_path, monkeypatch):
    # Regression the owner explicitly asked for alongside the https
    # refusal: an allowlisted scheme must still classify remote and
    # dispatch normally, unaffected by the scheme-agnostic classifier
    # change (gs/gcs weren't touched by the fix — only the sniff that
    # decides "does this need validate_remote_uri at all" changed).
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(4)])

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        assert builder == "json"
        if str(data_files).startswith("gs://"):
            return _FakeIterableDataset([{"text": f"B-{i}"} for i in range(3)])
        return _FakeIterableDataset(_rows_from_jsonl(data_files))

    monkeypatch.setattr(sys.modules["datasets"], "load_dataset", fake_load_dataset)

    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), "gs://bucket/b.jsonl"],
        interleave="concat",
        streaming=True,
    )
    result = load_dataset(cfg.data)
    texts = [row["text"] for row in result["train"]]
    assert len(texts) == 7
    assert any(t.startswith("A-") for t in texts)
    assert any(t.startswith("B-") for t in texts)


def test_single_https_train_rejected_by_name(tmp_path):
    # Consistency fix, same root cause: a single (non-list) data.train
    # https:// string used to fall through load_dataset()'s dispatch to
    # the plain local-file branch (FileNotFoundError on the mangled path)
    # instead of reaching _load_remote_dataset's validate_remote_uri check.
    # It must now be refused by name, the same as the list-path case.
    cfg = _cfg(tmp_path, train="https://example.com/data.jsonl", format="plaintext")
    with pytest.raises(ValueError, match="not in allowlist"):
        load_dataset(cfg.data)


@pytest.mark.parametrize(
    "suffix,expected_builder",
    [(".csv", "csv"), (".parquet", "parquet"), (".txt", "text")],
)
def test_streaming_builder_matches_file_suffix(tmp_path, monkeypatch, suffix, expected_builder):
    # MEDIUM — the streaming path used to hardcode the "json" builder for
    # every entry regardless of its actual type, breaking csv/txt/parquet
    # the moment data.streaming: true was set (they work fine non-streaming,
    # via load_raw_data's SUPPORTED_EXTENSIONS dispatch). The builder must
    # now be chosen per entry from its suffix (_streaming_builder_for),
    # matching that same dispatch instead of always assuming JSON.
    recorded: list = []
    (tmp_path / f"a{suffix}").write_text("placeholder", encoding="utf-8")
    (tmp_path / f"b{suffix}").write_text("placeholder", encoding="utf-8")

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        recorded.append(builder)
        return _FakeIterableDataset([{"text": "row"}])

    fake_module = type(sys)("datasets")
    fake_module.load_dataset = fake_load_dataset
    fake_module.concatenate_datasets = lambda streams: _FakeIterableDataset(
        [row for s in streams for row in s]
    )
    fake_module.interleave_datasets = lambda streams, **kw: _FakeIterableDataset(
        [row for s in streams for row in s]
    )
    monkeypatch.setitem(sys.modules, "datasets", fake_module)

    cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / f"a{suffix}"), str(tmp_path / f"b{suffix}")],
        interleave="concat",
        streaming=True,
        format="plaintext",
    )
    load_dataset(cfg.data)
    assert recorded == [expected_builder, expected_builder]


def test_streaming_unsupported_extension_rejected_by_name(tmp_path, monkeypatch):
    # An extension outside SUPPORTED_EXTENSIONS must refuse by name (mirrors
    # load_raw_data's "Unsupported file format" refusal for the non-streaming
    # path) rather than reach hf_load and fail with an opaque Arrow error.
    #
    # Uses a REMOTE entry for the bad extension, not a local one: after the
    # #468-review classifier fix (local requires suffix in
    # SUPPORTED_EXTENSIONS), a local ".xyz" entry classifies as 'hub', so
    # pairing it with a local ".jsonl" entry would hit the schema-level
    # "not a mix" refusal before ever reaching the loader — a 'remote' +
    # 'local' pair is not a mix, so this still exercises
    # _streaming_builder_for's refusal as intended.
    _write_jsonl(tmp_path / "b.jsonl", [f"B-{i}" for i in range(2)])

    reached: list = []

    def fake_load_dataset(builder, data_files=None, split=None, streaming=False):
        reached.append((builder, data_files))
        return _FakeIterableDataset([{"text": "unused"}])

    fake_module = type(sys)("datasets")
    fake_module.load_dataset = fake_load_dataset
    fake_module.concatenate_datasets = lambda streams: _FakeIterableDataset(
        [row for s in streams for row in s]
    )
    fake_module.interleave_datasets = lambda streams, **kw: _FakeIterableDataset(
        [row for s in streams for row in s]
    )
    monkeypatch.setitem(sys.modules, "datasets", fake_module)

    cfg = _cfg(
        tmp_path,
        train=["s3://bucket/a.xyz", str(tmp_path / "b.jsonl")],
        interleave="concat",
        streaming=True,
    )
    with pytest.raises(ValueError, match=r"\.xyz"):
        load_dataset(cfg.data)
    assert reached == []


# ---------------------------------------------------------------------------
# THE core acceptance test: one config, run both ways, proportions compared.
# ---------------------------------------------------------------------------


def test_probs_proportions_match_between_local_and_streaming_paths(tmp_path, monkeypatch):
    """Issue #459's own acceptance wording: "a test that runs one config
    both ways and compares the resulting proportions, not two tests that
    each pass in isolation."
    """
    calls: list = []
    _install_fake_streaming_datasets(monkeypatch, calls)
    # Large-enough pools that the weighted-without-replacement fake
    # converges close to the requested ratio before either source runs out.
    _write_jsonl(tmp_path / "a.jsonl", [f"A-{i}" for i in range(400)])
    _write_jsonl(tmp_path / "b.jsonl", [f"B-{i}" for i in range(400)])
    probs = [0.7, 0.3]

    local_cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")],
        interleave={"strategy": "probs", "probs": probs},
        streaming=False,
    )
    streaming_cfg = _cfg(
        tmp_path,
        train=[str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")],
        interleave={"strategy": "probs", "probs": probs},
        streaming=True,
    )

    local_result = load_dataset(local_cfg.data)
    streaming_result = load_dataset(streaming_cfg.data)

    local_texts = [row["text"] for row in local_result["train"]]
    streaming_texts = [row["text"] for row in streaming_result["train"]]

    local_a_ratio = sum(t.startswith("A-") for t in local_texts) / len(local_texts)
    streaming_a_ratio = sum(t.startswith("A-") for t in streaming_texts) / len(streaming_texts)

    assert local_a_ratio == pytest.approx(0.7, abs=1e-6)
    assert streaming_a_ratio == pytest.approx(0.7, abs=0.05)
    assert local_a_ratio == pytest.approx(streaming_a_ratio, abs=0.05)


# ---------------------------------------------------------------------------
# Loader-level, hub: combining + validation-split precedence
# ---------------------------------------------------------------------------


def _install_fake_hub_datasets(monkeypatch, registry: dict):
    def fake_load_dataset(name):
        return registry[name]

    fake_module = type(sys)("datasets")
    fake_module.load_dataset = fake_load_dataset
    monkeypatch.setitem(sys.modules, "datasets", fake_module)


def test_hub_list_concat_combines_all_entries(tmp_path, monkeypatch):
    registry = {
        "org/a": {"train": [{"text": f"A-{i}"} for i in range(4)]},
        "org/b": {"train": [{"text": f"B-{i}"} for i in range(3)]},
    }
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(tmp_path, train=["org/a", "org/b"], interleave="concat")
    result = load_dataset(cfg.data)
    texts = [row["text"] for row in result["train"]]
    assert len(texts) == 7
    assert sum(t.startswith("A-") for t in texts) == 4
    assert sum(t.startswith("B-") for t in texts) == 3


def test_hub_list_with_dotted_names_loads_via_hub_path(tmp_path, monkeypatch):
    # #468 review finding, at the loader level: if teknium/OpenHermes-2.5
    # were misclassified 'local' (pre-fix: Path.suffix == ".5" is
    # non-empty), load_dataset would try to open it as a local file and
    # raise FileNotFoundError instead of reaching the hub loader below.
    registry = {
        "teknium/OpenHermes-2.5": {"train": [{"text": f"A-{i}"} for i in range(4)]},
        "mlfoundations/dclm-baseline-1.0": {"train": [{"text": f"B-{i}"} for i in range(3)]},
    }
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(
        tmp_path,
        train=["teknium/OpenHermes-2.5", "mlfoundations/dclm-baseline-1.0"],
        interleave="concat",
    )
    result = load_dataset(cfg.data)
    texts = [row["text"] for row in result["train"]]
    assert len(texts) == 7
    assert sum(t.startswith("A-") for t in texts) == 4
    assert sum(t.startswith("B-") for t in texts) == 3


def test_hub_list_probs_not_silently_ignored(tmp_path, monkeypatch):
    # The #443-lesson test, applied to the hub path: fails if the key is
    # ignored — before #459, this shape refused at parse time; if the
    # loader silently dropped interleave, this would collapse to one
    # dataset's rows instead of the apportioned mixture.
    registry = {
        "org/a": {"train": [{"text": f"A-{i}"} for i in range(30)]},
        "org/b": {"train": [{"text": f"B-{i}"} for i in range(10)]},
    }
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(
        tmp_path,
        train=["org/a", "org/b"],
        interleave={"strategy": "probs", "probs": [0.75, 0.25]},
    )
    result = load_dataset(cfg.data)
    texts = [row["text"] for row in result["train"]]
    assert len(texts) == 40
    assert sum(t.startswith("A-") for t in texts) == 30
    assert sum(t.startswith("B-") for t in texts) == 10


def test_hub_validation_split_combined_when_every_entry_has_one(tmp_path, monkeypatch):
    registry = {
        "org/a": {
            "train": [{"text": f"A-{i}"} for i in range(4)],
            "validation": [{"text": f"Aval-{i}"} for i in range(2)],
        },
        "org/b": {
            "train": [{"text": f"B-{i}"} for i in range(4)],
            "validation": [{"text": f"Bval-{i}"} for i in range(2)],
        },
    }
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(tmp_path, train=["org/a", "org/b"], interleave="concat", val_split=0.5)
    result = load_dataset(cfg.data)
    val_texts = [row["text"] for row in result["val"]]
    assert len(val_texts) == 4
    assert sum(t.startswith("Aval-") for t in val_texts) == 2
    assert sum(t.startswith("Bval-") for t in val_texts) == 2
    # val_split (0.5) must NOT have been applied — hub splits won.
    assert len(result["train"]) == 8


def test_hub_validation_split_ignored_when_only_some_entries_have_one(tmp_path, monkeypatch):
    registry = {
        "org/a": {
            "train": [{"text": f"A-{i}"} for i in range(8)],
            "validation": [{"text": f"Aval-{i}"} for i in range(2)],
        },
        "org/b": {"train": [{"text": f"B-{i}"} for i in range(2)]},
    }
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(tmp_path, train=["org/a", "org/b"], interleave="concat", val_split=0.2)
    result = load_dataset(cfg.data)
    val_texts = [row["text"] for row in result["val"]]
    # The hub "validation" split (Aval-*) must be ignored entirely — a
    # partial hub split is not a decided mixture — and val_split applied
    # to the combined 10 train rows instead.
    assert not any(t.startswith("Aval-") for t in val_texts)
    combined_len = len(result["train"]) + len(result["val"])
    assert combined_len == 10
    assert len(result["val"]) == int(10 * 0.2)


def test_hub_partial_val_split_fallback_over_has_no_duplicate_row_across_train_and_val(
    tmp_path, monkeypatch
):
    # #680, hub path: the val_split fallback (only some entries have a hub
    # validation split, so it is ignored and val_split derives val instead)
    # goes through the same "over" cycling as the local path, and had the
    # same bug: it sliced the already-padded combined rows instead of
    # carving val out of each source's own unique rows first.
    registry = {
        "org/a": {
            "train": [{"text": f"A-{i}"} for i in range(8)],
            "validation": [{"text": f"Aval-{i}"} for i in range(2)],
        },
        "org/b": {"train": [{"text": f"B-{i}"} for i in range(2)]},
    }
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(tmp_path, train=["org/a", "org/b"], interleave="over", val_split=0.1)
    result = load_dataset(cfg.data)
    train_texts = {row["text"] for row in result["train"]}
    val_texts = {row["text"] for row in result["val"]}
    assert val_texts, "val must be non-empty, or the disjointness assertion is vacuous"
    assert not (train_texts & val_texts)
    assert not any(t.startswith("Aval-") for t in val_texts)


def test_hub_dataset_missing_train_split_still_raises(tmp_path, monkeypatch):
    registry = {
        "org/a": {"train": [{"text": "A-0"}]},
        "org/b": {"validation": [{"text": "Bval-0"}]},
    }
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(tmp_path, train=["org/a", "org/b"], interleave="concat")
    with pytest.raises(ValueError, match="no 'train' split"):
        load_dataset(cfg.data)


def test_load_hf_dataset_single_name_unchanged_by_refactor(tmp_path, monkeypatch):
    # Golden test for the _load_one_hub_dataset factoring — the single-name
    # branch (load_dataset() on a bare string, not a list) must produce
    # byte-identical output to before #459's refactor.
    registry = {"org/solo": {"train": [{"text": "X-0"}, {"text": "X-1"}]}}
    _install_fake_hub_datasets(monkeypatch, registry)
    cfg = _cfg(tmp_path, train="org/solo")
    result = load_dataset(cfg.data)
    assert result == {"train": [{"text": "X-0"}, {"text": "X-1"}]}