Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
feat(provenance): bind streamed base loads to recorded commits (#2094)
Nurkhan Esenbek503aba54e78e
diff --git a/changelog.d/0.76.0/2094.changed.md b/changelog.d/0.76.0/2094.changed.mdnew file mode 100644index 00000000..4526d8a6--- /dev/null+++ b/changelog.d/0.76.0/2094.changed.md@@ -0,0 +1,7 @@+- **Bind base revisions in single-process HF layer streaming+ (#1902 by @kok-o in #2094).** Streamed config/skeleton and checkpoint downloads+ share the run's resolved commit, with revision-specific shard and weight-copy+ cache slots. The first pinned streamed run after upgrading re-shards and+ re-copies the base into the new `<slug>@<commit>` slots, costing disk space and+ time; old slots are left in place. Local/cache-path hints remain unverified;+ Unsloth, MLX and distributed loads remain unbound.diff --git a/docs/backends-and-ops.md b/docs/backends-and-ops.mdindex 8b5dfb0a..133aefad 100644--- a/docs/backends-and-ops.md+++ b/docs/backends-and-ops.md@@ -900,13 +900,23 @@ Every `soup train` run is automatically tracked in a local SQLite database (`~/.### Recorded base identity-For single-process resident Hugging Face loads on the Transformers backend, `soup train` resolves+For single-process Hugging Face loads on the Transformers backend (resident or+layer-streamed), `soup train` resolvesthe base's commit once, immediately before its first model/tokenizer/processor/configload, and passes that commit to the run's subsequent base loads. The original `base`name stays in the config and experiments row; unrelated teacher, reward and scorerrepositories do not receive its revision. `soup runs show` displays the revision andits source.+Layer streaming pins its tokenizer, architecture config, probe/final meta+skeletons and both checkpoint download passes. Known Hub commits use separate+shard and materialized-weight cache slots; repeated loads of one commit reuse+its slot. Local paths and unknown revisions retain the existing cache behavior.+The config, PEFT base name and run metadata continue to name the original base.+The first pinned streamed run after upgrading re-shards and re-copies the base+into the new `<slug>@<commit>` cache slots, costing disk space and time; old slots+are left in place.+The new nullable `runs.base_revision`, `runs.base_revision_source` and`runs.training_data_sha256` columns upgrade existing databases in place. Historicalrows stay unknown; Soup never fills them from today's Hub state or dataset bytes.@@ -931,7 +941,7 @@ Loading a local adapter folder directly also reuses that commit when its`adapter_config.json` names the same base and has no conflicting explicit revision;the tokenizer still comes from the adapter folder.-**First-stage boundaries:** Non-HF hubs, Unsloth, MLX, layer-streaming and distributed loads remain unbound and+**Remaining boundaries:** Non-HF hubs, Unsloth, MLX and distributed loads remain unbound andare recorded as such; a cache-path hint on any backend is still unverified. Standalonewrapper/sweep invocations do not automatically establish a run scope. Metadata iswritten for the final artifact, not each intermediate checkpoint. Other exportersdiff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.pyindex b002c6bf..6af7d701 100644--- a/src/soup_cli/commands/train.py+++ b/src/soup_cli/commands/train.py@@ -1913,7 +1913,7 @@ def train(base_identity = BaseRevision(run_base,binding_supported=(cfg.backend == "transformers" and hub_name == "hf"- and not cfg.training.stream_layers and not is_in_distributed()),+ and not is_in_distributed()),)with base_revision_scope(base_identity):trainer_wrapper = build_trainer(cfg, **trainer_kwargs)diff --git a/src/soup_cli/utils/layer_shard.py b/src/soup_cli/utils/layer_shard.pyindex 0944c9ea..9aa72a99 100644--- a/src/soup_cli/utils/layer_shard.py+++ b/src/soup_cli/utils/layer_shard.py@@ -813,8 +813,13 @@ def resolve_shard_dir(model: str, cache_dir: Optional[str] = None) -> str:an env var is operator config, not API input (mirrors spectrum_scan)."""from soup_cli.utils.paths import is_under+ from soup_cli.utils.run_provenance import base_revision_kwargsslug = model_slug(model)+ revision = base_revision_kwargs(model).get("revision")+ if revision:+ # '@' cannot occur in model_slug, so no model id aliases this slot.+ slug += "@" + revisionroot: str = _resolve_cache_root(cache_dir, is_under)chosen = os.path.join(root, slug)os.makedirs(chosen, exist_ok=True)diff --git a/src/soup_cli/utils/layer_stream_runtime.py b/src/soup_cli/utils/layer_stream_runtime.pyindex 2ce8422a..86af91df 100644--- a/src/soup_cli/utils/layer_stream_runtime.py+++ b/src/soup_cli/utils/layer_stream_runtime.py@@ -2563,6 +2563,7 @@ def build_meta_skeleton(from transformers import AutoConfig, AutoModelForCausalLMfrom soup_cli.utils.layer_shard import QUANT_NF4, SUPPORTED_STREAM_QUANTS+ from soup_cli.utils.run_provenance import base_revision_kwargsif quant not in SUPPORTED_STREAM_QUANTS:raise ValueError(@@ -2570,7 +2571,9 @@ def build_meta_skeleton(f"{', '.join(SUPPORTED_STREAM_QUANTS)}")torch_dtype = _torch_dtype(dtype)- config = AutoConfig.from_pretrained(model_id, trust_remote_code=trust_remote_code)+ config = AutoConfig.from_pretrained(+ model_id, trust_remote_code=trust_remote_code, **base_revision_kwargs(model_id),+ )config.use_cache = False_drop_fp8_source_quantization(config)quant_config = Nonediff --git a/src/soup_cli/utils/spectrum_scan.py b/src/soup_cli/utils/spectrum_scan.pyindex c8b4de16..29c41f22 100644--- a/src/soup_cli/utils/spectrum_scan.py+++ b/src/soup_cli/utils/spectrum_scan.py@@ -999,9 +999,12 @@ def _download_hub_checkpoint(model: str) -> str:select_checkpoint_files,)from soup_cli.utils.hubs import snapshot_download+ from soup_cli.utils.run_provenance import base_revision_kwargs+ revision_kwargs = base_revision_kwargs(model)source = snapshot_download(- model, cache_dir=None, allow_patterns=[*hub_allow_patterns(None), "*.json"]+ model, cache_dir=None, allow_patterns=[*hub_allow_patterns(None), "*.json"],+ **revision_kwargs,)real = os.path.realpath(source)selected = select_checkpoint_files(os.listdir(real), read_index_weight_map(real))@@ -1013,6 +1016,7 @@ def _download_hub_checkpoint(model: str) -> str:cache_dir=None,allow_patterns=[*allow, "*.json"],ignore_patterns=ignore or None,+ **revision_kwargs,)return source@@ -1066,8 +1070,15 @@ def _hf_snapshot_plan(slot: Optional[str] = None,) -> ModelWeightsPlan:"""Plan the regular-file copy of a symlinked HF snapshot (reused when current)."""+ from soup_cli.utils.run_provenance import base_revision_kwargs++ if slot is None:+ slot = model_slug(model)+ revision = base_revision_kwargs(model).get("revision")+ if revision:+ slot += "@" + revisionmaterialized = os.path.join(- resolve_cache_dir(), "weights", slot or model_slug(model)+ resolve_cache_dir(), "weights", slot)existing = _materialized_matches_hf_snapshot(source,diff --git a/tests/test_backend_base_revision.py b/tests/test_backend_base_revision.pynew file mode 100644index 00000000..f222eb8b--- /dev/null+++ b/tests/test_backend_base_revision.py@@ -0,0 +1,245 @@+"""Streamed base pins and caches, with unchanged Unsloth mapping as a control."""++from __future__ import annotations++import json+import os+import sys+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import Mock++import pytest++from soup_cli.utils import run_provenance as provenance++BASE = "owner/base"+SHA = "a" * 40+++def moving_hub(monkeypatch):+ monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)+ monkeypatch.delenv("UNSLOTH_USE_MODELSCOPE", raising=False)+ probe = Mock(side_effect=[SimpleNamespace(sha=SHA), SimpleNamespace(sha="b" * 40)])+ monkeypatch.setattr(provenance, "get_hf_api", lambda: SimpleNamespace(model_info=probe))+ return probe+++def test_stream_setup_pins_tokenizer_and_architecture_config(monkeypatch, moving_hub):+ from transformers import AutoConfig, AutoTokenizer++ from soup_cli.config.schema import SoupConfig+ from soup_cli.trainer.sft import SFTTrainerWrapper++ calls = []++ def tokenizer(model_id, *, revision=None, **kwargs):+ assert model_id == BASE and revision == SHA+ calls.append("tokenizer")+ return SimpleNamespace(pad_token=None, eos_token="eos")++ def config(model_id, *, revision=None, **kwargs):+ assert model_id == BASE and revision == SHA+ calls.append("config")+ return object()++ class AfterConfigError(Exception):+ pass++ def stop_before_allocation(config):+ raise AfterConfigError++ monkeypatch.setattr(AutoTokenizer, "from_pretrained", tokenizer)+ monkeypatch.setattr(AutoConfig, "from_pretrained", config)+ monkeypatch.setattr("soup_cli.utils.layer_stream.stream_arch_of", stop_before_allocation)+ cfg = SoupConfig(base=BASE, data={"train": "unused.jsonl"},+ training={"quantization": "none", "stream_layers": True, "batch_size": 1})+ wrapper = SFTTrainerWrapper(cfg, device="cpu")+ identity = provenance.BaseRevision(BASE)+ with provenance.base_revision_scope(identity):+ with pytest.raises(AfterConfigError):+ wrapper._setup_streaming_transformers(cfg, cfg.training)+ assert identity.revision == SHA+ assert calls == ["tokenizer", "config"]+ moving_hub.assert_called_once()++[email protected]("second_call", [False, True])+def test_stream_skeleton_and_weight_downloads_share_one_commit(+ tmp_path, monkeypatch, moving_hub, second_call,+):+ from transformers import AutoConfig++ from soup_cli.utils import hubs, spectrum_scan+ from soup_cli.utils.layer_shard import resolve_shard_dir, shard_checkpoint+ from soup_cli.utils.layer_stream_runtime import build_meta_skeleton+ from tests._tiny_hf_models import tiny_model_dir++ local = tiny_model_dir("llama")+ snapshot = tmp_path / "snapshot"+ snapshot.mkdir()+ # An index with a nonstandard weight spelling exercises the second fetch too.+ if second_call:+ (snapshot / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"x": "part.safetensors"}}), encoding="utf-8",+ )+ calls = []++ def download(model_id, *, revision=None, **kwargs):+ assert model_id == BASE+ assert revision == SHA, "a download must not follow the moving default branch"+ calls.append(kwargs)+ if not second_call or len(calls) == 2:+ import shutil++ shutil.copyfile(Path(local) / "model.safetensors",+ snapshot / ("part.safetensors" if second_call else "model.safetensors"))+ return str(snapshot)++ original_config = AutoConfig.from_pretrained+ configs = []++ def config(model_id, **kwargs):+ assert model_id == BASE+ assert kwargs.pop("revision", None) == SHA+ configs.append(model_id)+ return original_config(local, **kwargs)++ monkeypatch.setattr(hubs, "snapshot_download", download)+ monkeypatch.setattr(AutoConfig, "from_pretrained", config)+ identity = provenance.BaseRevision(BASE)+ with provenance.base_revision_scope(identity):+ # Probe and final skeleton both read the same source config.+ for _ in range(2):+ skeleton = build_meta_skeleton(BASE, dtype="float32")+ assert skeleton.name_or_path == BASE+ assert all(parameter.is_meta for parameter in skeleton.parameters())+ weights = spectrum_scan.resolve_model_weights(BASE)+ shards = resolve_shard_dir(BASE, str(tmp_path / "shards"))+ index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")+ assert index is not None and os.path.isdir(shards)+ assert configs == [BASE, BASE]+ assert len(calls) == (2 if second_call else 1)+ assert moving_hub.call_count == 1+ assert identity.metadata("run")["base_revision"] == SHA+++def test_shard_slots_separate_commits_and_leave_other_models_unchanged(tmp_path, moving_hub):+ from soup_cli.utils.layer_shard import resolve_shard_dir++ cache = str(tmp_path / "cache")+ unscoped = resolve_shard_dir(BASE, cache)+ other = resolve_shard_dir("owner/teacher", cache)+ slots = []+ for sha in (SHA, "b" * 40):+ with provenance.base_revision_scope(provenance.BaseRevision(BASE)):+ slot = resolve_shard_dir(BASE, cache)+ assert Path(slot).name.endswith("@" + sha)+ assert resolve_shard_dir(BASE, cache) == slot+ assert resolve_shard_dir("owner/teacher", cache) == other+ assert os.path.commonpath([os.path.realpath(slot), os.path.realpath(cache)]) == cache+ slots.append(slot)+ assert len(set([unscoped, *slots])) == 3+ assert moving_hub.call_count == 2+++def test_materialized_weight_slots_separate_pinned_commits(tmp_path, monkeypatch, moving_hub):+ from soup_cli.utils import spectrum_scan++ monkeypatch.setenv("SOUP_SPECTRUM_CACHE_DIR", str(tmp_path / "cache"))+ source = tmp_path / "snapshot"+ source.mkdir()+ slots = []+ # Use the planner directly: no Windows symlink privilege is required.+ for sha in (SHA, "b" * 40):+ with provenance.base_revision_scope(provenance.BaseRevision(BASE)):+ plan = spectrum_scan._hf_snapshot_plan(BASE, str(source), (), sha)+ assert Path(plan.weights_dir).name.endswith("@" + sha)+ slots.append(plan.weights_dir)+ assert slots[0] != slots[1]+ assert moving_hub.call_count == 2++[email protected]("quantization", ["none", "4bit"])+def test_unsloth_keeps_native_mapping_without_a_revision_override(+ monkeypatch, moving_hub, quantization,+):+ from soup_cli.utils.unsloth import load_model_and_tokenizer++ calls = []+ model = SimpleNamespace(config=SimpleNamespace(_name_or_path=BASE, _commit_hash=SHA))+ tokenizer = object()++ def load(*, model_name, revision=None, use_exact_model_name=False, **kwargs):+ # This receiver is an API-contract fixture, not an installed Unsloth run.+ # Native quantized mapping must remain enabled for official model IDs.+ assert model_name == BASE and use_exact_model_name is False+ assert revision is None+ assert kwargs["load_in_4bit"] is (quantization == "4bit")+ calls.append(model_name + "-bnb-4bit" if kwargs["load_in_4bit"] else model_name)+ return model, tokenizer++ peft = Mock(return_value=model)+ monkeypatch.setitem(sys.modules, "unsloth", SimpleNamespace(FastLanguageModel=SimpleNamespace(+ from_pretrained=load, get_peft_model=peft,+ )))+ identity = provenance.BaseRevision(BASE)+ with provenance.base_revision_scope(identity):+ assert load_model_and_tokenizer(BASE, 64, quantization=quantization) == (model, tokenizer)+ assert calls == [BASE + "-bnb-4bit" if quantization == "4bit" else BASE]+ assert peft.call_args.kwargs["use_gradient_checkpointing"] == "unsloth"+ moving_hub.assert_not_called()+ assert identity.revision is None+++def test_local_stream_cache_keeps_unverified_path_hint(tmp_path, monkeypatch, moving_hub):+ from soup_cli.utils.layer_shard import resolve_shard_dir++ local = tmp_path / "snapshots" / SHA+ local.mkdir(parents=True)+ base = str(local)+ cache = str(tmp_path / "cache")+ expected = resolve_shard_dir(base, cache)+ identity = provenance.BaseRevision(base)+ with provenance.base_revision_scope(identity):+ assert provenance.base_revision_kwargs(base) == {}+ assert resolve_shard_dir(base, cache) == expected+ assert identity.revision == SHA and identity.source == "cache_path"+ moving_hub.assert_not_called()++[email protected]("mode", ["offline", "failed"])+def test_unknown_stream_weights_keep_unpinned_download_and_cache(+ tmp_path, monkeypatch, moving_hub, mode,+):+ from soup_cli.utils import hubs, spectrum_scan+ from soup_cli.utils.layer_shard import resolve_shard_dir++ if mode == "offline":+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")+ else:+ moving_hub.side_effect = OSError("metadata unavailable")+ snapshot = tmp_path / "snapshot"+ snapshot.mkdir()+ (snapshot / "model.safetensors").write_bytes(b"existing local weight fixture")+ calls = []++ def download(model_id, **kwargs):+ assert model_id == BASE+ assert "revision" not in kwargs+ calls.append(kwargs)+ return str(snapshot)++ monkeypatch.setattr(hubs, "snapshot_download", download)+ cache = str(tmp_path / "cache")+ original_slot = resolve_shard_dir(BASE, cache)+ identity = provenance.BaseRevision(BASE)+ with provenance.base_revision_scope(identity):+ assert spectrum_scan.resolve_model_weights(BASE) == str(snapshot)+ assert resolve_shard_dir(BASE, cache) == original_slot+ assert len(calls) == 1+ assert identity.revision is None+ assert identity.source == ("offline" if mode == "offline" else "resolution_failed")+ assert moving_hub.call_count == (0 if mode == "offline" else 1)diff --git a/tests/test_base_revision_loading.py b/tests/test_base_revision_loading.pyindex 937303c6..75b93c95 100644--- a/tests/test_base_revision_loading.py+++ b/tests/test_base_revision_loading.py@@ -225,7 +225,9 @@ def test_dry_run_does_not_capture_training_identity(tmp_path, monkeypatch):capture.assert_not_called()[email protected]("mode", ["bound", "offline", "distributed", "non-hf"])[email protected]("mode", [+ "bound", "streaming", "unsloth", "offline", "distributed", "non-hf",+])@pytest.mark.parametrize("source_state", ["unchanged", "changed", "deleted"])def test_train_cli_records_identity_and_writes_final_artifact(tmp_path, monkeypatch, moving_hub, mode, source_state,@@ -244,6 +246,7 @@ def test_train_cli_records_identity_and_writes_final_artifact(monkeypatch.setattr("soup_cli.utils.gpu.detect_device", lambda: ("cpu", "CPU"))monkeypatch.setattr("soup_cli.utils.gpu.get_gpu_info", lambda: {})alternate_snapshot = None+ bound = mode in {"bound", "streaming"}def alternate_hub(hub, model_id, *, local_dir):nonlocal alternate_snapshot@@ -261,15 +264,17 @@ def test_train_cli_records_identity_and_writes_final_artifact('{"instruction":"hi", "output":"hello"}\n', encoding="utf-8",)(tmp_path / "soup.yaml").write_text(- f"base: {BASE}\ndata: {{train: data.jsonl, format: alpaca, val_split: 0}}\n"+ f"base: {BASE}\nbackend: {'unsloth' if mode == 'unsloth' else 'transformers'}\n"+ "data: {train: data.jsonl, format: alpaca, val_split: 0}\n""training: {epochs: 1, quantization: none, batch_size: 1, "+ f"stream_layers: {str(mode == 'streaming').lower()}, "f"hub: {'modelscope' if mode == 'non-hf' else 'hf'}}}\noutput: final\n",encoding="utf-8",)import hashlibstored_hash = hashlib.sha256((tmp_path / "data.jsonl").read_bytes()).hexdigest()- expected = {"revision": REVISION} if mode == "bound" else {}+ expected = {"revision": REVISION} if bound else {}class Trainer:def setup(self, dataset):@@ -303,15 +308,16 @@ def test_train_cli_records_identity_and_writes_final_artifact(assert row["status"] == "completed"assert row["base_model"] == BASEassert json.loads(row["config_json"])["base"] == BASE- assert row["base_revision"] == (REVISION if mode == "bound" else None)+ assert row["base_revision"] == (REVISION if bound else None)assert row["base_revision_source"] == {- "bound": "hub_commit", "offline": "offline", "distributed": "unsupported_backend",+ "bound": "hub_commit", "streaming": "hub_commit", "unsloth": "unsupported_backend",+ "offline": "offline", "distributed": "unsupported_backend","non-hf": "unsupported_backend",}[mode]metadata = provenance.read_run_provenance(output)assert metadata["run_id"] == row["run_id"]assert metadata["base_revision"] == row["base_revision"]assert metadata["training_data_sha256"] == stored_hash- assert moving_hub.call_count == (1 if mode == "bound" else 0)+ assert moving_hub.call_count == (1 if bound else 0)assert provenance.base_revision_kwargs(BASE) == {}tracker.close()