Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

feat(provenance): bind streamed base loads to recorded commits (#2094)

Nurkhan Esenbek503aba54e78e
diff --git a/changelog.d/0.76.0/2094.changed.md b/changelog.d/0.76.0/2094.changed.md
new file mode 100644
index 00000000..4526d8a6
--- /dev/null
+++ b/changelog.d/0.76.0/2094.changed.md
@@ -0,0 +1,7 @@
+- **Bind base revisions in single-process HF layer streaming
+ (#1902 by @kok-o in #2094).** Streamed config/skeleton and checkpoint downloads
+ share the run's resolved commit, with revision-specific shard and weight-copy
+ cache slots. The first pinned streamed run after upgrading re-shards and
+ re-copies the base into the new `<slug>@<commit>` slots, costing disk space and
+ time; old slots are left in place. Local/cache-path hints remain unverified;
+ Unsloth, MLX and distributed loads remain unbound.
diff --git a/docs/backends-and-ops.md b/docs/backends-and-ops.md
index 8b5dfb0a..133aefad 100644
--- a/docs/backends-and-ops.md
+++ b/docs/backends-and-ops.md
@@ -900,13 +900,23 @@ Every `soup train` run is automatically tracked in a local SQLite database (`~/.
### Recorded base identity
-For single-process resident Hugging Face loads on the Transformers backend, `soup train` resolves
+For single-process Hugging Face loads on the Transformers backend (resident or
+layer-streamed), `soup train` resolves
the base's commit once, immediately before its first model/tokenizer/processor/config
load, and passes that commit to the run's subsequent base loads. The original `base`
name stays in the config and experiments row; unrelated teacher, reward and scorer
repositories do not receive its revision. `soup runs show` displays the revision and
its source.
+Layer streaming pins its tokenizer, architecture config, probe/final meta
+skeletons and both checkpoint download passes. Known Hub commits use separate
+shard and materialized-weight cache slots; repeated loads of one commit reuse
+its slot. Local paths and unknown revisions retain the existing cache behavior.
+The config, PEFT base name and run metadata continue to name the original base.
+The first pinned streamed run after upgrading re-shards and re-copies the base
+into the new `<slug>@<commit>` cache slots, costing disk space and time; old slots
+are left in place.
+
The new nullable `runs.base_revision`, `runs.base_revision_source` and
`runs.training_data_sha256` columns upgrade existing databases in place. Historical
rows stay unknown; Soup never fills them from today's Hub state or dataset bytes.
@@ -931,7 +941,7 @@ Loading a local adapter folder directly also reuses that commit when its
`adapter_config.json` names the same base and has no conflicting explicit revision;
the tokenizer still comes from the adapter folder.
-**First-stage boundaries:** Non-HF hubs, Unsloth, MLX, layer-streaming and distributed loads remain unbound and
+**Remaining boundaries:** Non-HF hubs, Unsloth, MLX and distributed loads remain unbound and
are recorded as such; a cache-path hint on any backend is still unverified. Standalone
wrapper/sweep invocations do not automatically establish a run scope. Metadata is
written for the final artifact, not each intermediate checkpoint. Other exporters
diff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.py
index b002c6bf..6af7d701 100644
--- a/src/soup_cli/commands/train.py
+++ b/src/soup_cli/commands/train.py
@@ -1913,7 +1913,7 @@ def train(
base_identity = BaseRevision(
run_base,
binding_supported=(cfg.backend == "transformers" and hub_name == "hf"
- and not cfg.training.stream_layers and not is_in_distributed()),
+ and not is_in_distributed()),
)
with base_revision_scope(base_identity):
trainer_wrapper = build_trainer(cfg, **trainer_kwargs)
diff --git a/src/soup_cli/utils/layer_shard.py b/src/soup_cli/utils/layer_shard.py
index 0944c9ea..9aa72a99 100644
--- a/src/soup_cli/utils/layer_shard.py
+++ b/src/soup_cli/utils/layer_shard.py
@@ -813,8 +813,13 @@ def resolve_shard_dir(model: str, cache_dir: Optional[str] = None) -> str:
an env var is operator config, not API input (mirrors spectrum_scan).
"""
from soup_cli.utils.paths import is_under
+ from soup_cli.utils.run_provenance import base_revision_kwargs
slug = model_slug(model)
+ revision = base_revision_kwargs(model).get("revision")
+ if revision:
+ # '@' cannot occur in model_slug, so no model id aliases this slot.
+ slug += "@" + revision
root: str = _resolve_cache_root(cache_dir, is_under)
chosen = os.path.join(root, slug)
os.makedirs(chosen, exist_ok=True)
diff --git a/src/soup_cli/utils/layer_stream_runtime.py b/src/soup_cli/utils/layer_stream_runtime.py
index 2ce8422a..86af91df 100644
--- a/src/soup_cli/utils/layer_stream_runtime.py
+++ b/src/soup_cli/utils/layer_stream_runtime.py
@@ -2563,6 +2563,7 @@ def build_meta_skeleton(
from transformers import AutoConfig, AutoModelForCausalLM
from soup_cli.utils.layer_shard import QUANT_NF4, SUPPORTED_STREAM_QUANTS
+ from soup_cli.utils.run_provenance import base_revision_kwargs
if quant not in SUPPORTED_STREAM_QUANTS:
raise ValueError(
@@ -2570,7 +2571,9 @@ def build_meta_skeleton(
f"{', '.join(SUPPORTED_STREAM_QUANTS)}"
)
torch_dtype = _torch_dtype(dtype)
- config = AutoConfig.from_pretrained(model_id, trust_remote_code=trust_remote_code)
+ config = AutoConfig.from_pretrained(
+ model_id, trust_remote_code=trust_remote_code, **base_revision_kwargs(model_id),
+ )
config.use_cache = False
_drop_fp8_source_quantization(config)
quant_config = None
diff --git a/src/soup_cli/utils/spectrum_scan.py b/src/soup_cli/utils/spectrum_scan.py
index c8b4de16..29c41f22 100644
--- a/src/soup_cli/utils/spectrum_scan.py
+++ b/src/soup_cli/utils/spectrum_scan.py
@@ -999,9 +999,12 @@ def _download_hub_checkpoint(model: str) -> str:
select_checkpoint_files,
)
from soup_cli.utils.hubs import snapshot_download
+ from soup_cli.utils.run_provenance import base_revision_kwargs
+ revision_kwargs = base_revision_kwargs(model)
source = snapshot_download(
- model, cache_dir=None, allow_patterns=[*hub_allow_patterns(None), "*.json"]
+ model, cache_dir=None, allow_patterns=[*hub_allow_patterns(None), "*.json"],
+ **revision_kwargs,
)
real = os.path.realpath(source)
selected = select_checkpoint_files(os.listdir(real), read_index_weight_map(real))
@@ -1013,6 +1016,7 @@ def _download_hub_checkpoint(model: str) -> str:
cache_dir=None,
allow_patterns=[*allow, "*.json"],
ignore_patterns=ignore or None,
+ **revision_kwargs,
)
return source
@@ -1066,8 +1070,15 @@ def _hf_snapshot_plan(
slot: Optional[str] = None,
) -> ModelWeightsPlan:
"""Plan the regular-file copy of a symlinked HF snapshot (reused when current)."""
+ from soup_cli.utils.run_provenance import base_revision_kwargs
+
+ if slot is None:
+ slot = model_slug(model)
+ revision = base_revision_kwargs(model).get("revision")
+ if revision:
+ slot += "@" + revision
materialized = os.path.join(
- resolve_cache_dir(), "weights", slot or model_slug(model)
+ resolve_cache_dir(), "weights", slot
)
existing = _materialized_matches_hf_snapshot(
source,
diff --git a/tests/test_backend_base_revision.py b/tests/test_backend_base_revision.py
new file mode 100644
index 00000000..f222eb8b
--- /dev/null
+++ b/tests/test_backend_base_revision.py
@@ -0,0 +1,245 @@
+"""Streamed base pins and caches, with unchanged Unsloth mapping as a control."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+from soup_cli.utils import run_provenance as provenance
+
+BASE = "owner/base"
+SHA = "a" * 40
+
+
+def moving_hub(monkeypatch):
+ monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)
+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)
+ monkeypatch.delenv("UNSLOTH_USE_MODELSCOPE", raising=False)
+ probe = Mock(side_effect=[SimpleNamespace(sha=SHA), SimpleNamespace(sha="b" * 40)])
+ monkeypatch.setattr(provenance, "get_hf_api", lambda: SimpleNamespace(model_info=probe))
+ return probe
+
+
+def test_stream_setup_pins_tokenizer_and_architecture_config(monkeypatch, moving_hub):
+ from transformers import AutoConfig, AutoTokenizer
+
+ from soup_cli.config.schema import SoupConfig
+ from soup_cli.trainer.sft import SFTTrainerWrapper
+
+ calls = []
+
+ def tokenizer(model_id, *, revision=None, **kwargs):
+ assert model_id == BASE and revision == SHA
+ calls.append("tokenizer")
+ return SimpleNamespace(pad_token=None, eos_token="eos")
+
+ def config(model_id, *, revision=None, **kwargs):
+ assert model_id == BASE and revision == SHA
+ calls.append("config")
+ return object()
+
+ class AfterConfigError(Exception):
+ pass
+
+ def stop_before_allocation(config):
+ raise AfterConfigError
+
+ monkeypatch.setattr(AutoTokenizer, "from_pretrained", tokenizer)
+ monkeypatch.setattr(AutoConfig, "from_pretrained", config)
+ monkeypatch.setattr("soup_cli.utils.layer_stream.stream_arch_of", stop_before_allocation)
+ cfg = SoupConfig(base=BASE, data={"train": "unused.jsonl"},
+ training={"quantization": "none", "stream_layers": True, "batch_size": 1})
+ wrapper = SFTTrainerWrapper(cfg, device="cpu")
+ identity = provenance.BaseRevision(BASE)
+ with provenance.base_revision_scope(identity):
+ with pytest.raises(AfterConfigError):
+ wrapper._setup_streaming_transformers(cfg, cfg.training)
+ assert identity.revision == SHA
+ assert calls == ["tokenizer", "config"]
+ moving_hub.assert_called_once()
+
+
[email protected]("second_call", [False, True])
+def test_stream_skeleton_and_weight_downloads_share_one_commit(
+ tmp_path, monkeypatch, moving_hub, second_call,
+):
+ from transformers import AutoConfig
+
+ from soup_cli.utils import hubs, spectrum_scan
+ from soup_cli.utils.layer_shard import resolve_shard_dir, shard_checkpoint
+ from soup_cli.utils.layer_stream_runtime import build_meta_skeleton
+ from tests._tiny_hf_models import tiny_model_dir
+
+ local = tiny_model_dir("llama")
+ snapshot = tmp_path / "snapshot"
+ snapshot.mkdir()
+ # An index with a nonstandard weight spelling exercises the second fetch too.
+ if second_call:
+ (snapshot / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"x": "part.safetensors"}}), encoding="utf-8",
+ )
+ calls = []
+
+ def download(model_id, *, revision=None, **kwargs):
+ assert model_id == BASE
+ assert revision == SHA, "a download must not follow the moving default branch"
+ calls.append(kwargs)
+ if not second_call or len(calls) == 2:
+ import shutil
+
+ shutil.copyfile(Path(local) / "model.safetensors",
+ snapshot / ("part.safetensors" if second_call else "model.safetensors"))
+ return str(snapshot)
+
+ original_config = AutoConfig.from_pretrained
+ configs = []
+
+ def config(model_id, **kwargs):
+ assert model_id == BASE
+ assert kwargs.pop("revision", None) == SHA
+ configs.append(model_id)
+ return original_config(local, **kwargs)
+
+ monkeypatch.setattr(hubs, "snapshot_download", download)
+ monkeypatch.setattr(AutoConfig, "from_pretrained", config)
+ identity = provenance.BaseRevision(BASE)
+ with provenance.base_revision_scope(identity):
+ # Probe and final skeleton both read the same source config.
+ for _ in range(2):
+ skeleton = build_meta_skeleton(BASE, dtype="float32")
+ assert skeleton.name_or_path == BASE
+ assert all(parameter.is_meta for parameter in skeleton.parameters())
+ weights = spectrum_scan.resolve_model_weights(BASE)
+ shards = resolve_shard_dir(BASE, str(tmp_path / "shards"))
+ index = shard_checkpoint(weights, shards, dtype="float32", arch="llama")
+ assert index is not None and os.path.isdir(shards)
+ assert configs == [BASE, BASE]
+ assert len(calls) == (2 if second_call else 1)
+ assert moving_hub.call_count == 1
+ assert identity.metadata("run")["base_revision"] == SHA
+
+
+def test_shard_slots_separate_commits_and_leave_other_models_unchanged(tmp_path, moving_hub):
+ from soup_cli.utils.layer_shard import resolve_shard_dir
+
+ cache = str(tmp_path / "cache")
+ unscoped = resolve_shard_dir(BASE, cache)
+ other = resolve_shard_dir("owner/teacher", cache)
+ slots = []
+ for sha in (SHA, "b" * 40):
+ with provenance.base_revision_scope(provenance.BaseRevision(BASE)):
+ slot = resolve_shard_dir(BASE, cache)
+ assert Path(slot).name.endswith("@" + sha)
+ assert resolve_shard_dir(BASE, cache) == slot
+ assert resolve_shard_dir("owner/teacher", cache) == other
+ assert os.path.commonpath([os.path.realpath(slot), os.path.realpath(cache)]) == cache
+ slots.append(slot)
+ assert len(set([unscoped, *slots])) == 3
+ assert moving_hub.call_count == 2
+
+
+def test_materialized_weight_slots_separate_pinned_commits(tmp_path, monkeypatch, moving_hub):
+ from soup_cli.utils import spectrum_scan
+
+ monkeypatch.setenv("SOUP_SPECTRUM_CACHE_DIR", str(tmp_path / "cache"))
+ source = tmp_path / "snapshot"
+ source.mkdir()
+ slots = []
+ # Use the planner directly: no Windows symlink privilege is required.
+ for sha in (SHA, "b" * 40):
+ with provenance.base_revision_scope(provenance.BaseRevision(BASE)):
+ plan = spectrum_scan._hf_snapshot_plan(BASE, str(source), (), sha)
+ assert Path(plan.weights_dir).name.endswith("@" + sha)
+ slots.append(plan.weights_dir)
+ assert slots[0] != slots[1]
+ assert moving_hub.call_count == 2
+
+
[email protected]("quantization", ["none", "4bit"])
+def test_unsloth_keeps_native_mapping_without_a_revision_override(
+ monkeypatch, moving_hub, quantization,
+):
+ from soup_cli.utils.unsloth import load_model_and_tokenizer
+
+ calls = []
+ model = SimpleNamespace(config=SimpleNamespace(_name_or_path=BASE, _commit_hash=SHA))
+ tokenizer = object()
+
+ def load(*, model_name, revision=None, use_exact_model_name=False, **kwargs):
+ # This receiver is an API-contract fixture, not an installed Unsloth run.
+ # Native quantized mapping must remain enabled for official model IDs.
+ assert model_name == BASE and use_exact_model_name is False
+ assert revision is None
+ assert kwargs["load_in_4bit"] is (quantization == "4bit")
+ calls.append(model_name + "-bnb-4bit" if kwargs["load_in_4bit"] else model_name)
+ return model, tokenizer
+
+ peft = Mock(return_value=model)
+ monkeypatch.setitem(sys.modules, "unsloth", SimpleNamespace(FastLanguageModel=SimpleNamespace(
+ from_pretrained=load, get_peft_model=peft,
+ )))
+ identity = provenance.BaseRevision(BASE)
+ with provenance.base_revision_scope(identity):
+ assert load_model_and_tokenizer(BASE, 64, quantization=quantization) == (model, tokenizer)
+ assert calls == [BASE + "-bnb-4bit" if quantization == "4bit" else BASE]
+ assert peft.call_args.kwargs["use_gradient_checkpointing"] == "unsloth"
+ moving_hub.assert_not_called()
+ assert identity.revision is None
+
+
+def test_local_stream_cache_keeps_unverified_path_hint(tmp_path, monkeypatch, moving_hub):
+ from soup_cli.utils.layer_shard import resolve_shard_dir
+
+ local = tmp_path / "snapshots" / SHA
+ local.mkdir(parents=True)
+ base = str(local)
+ cache = str(tmp_path / "cache")
+ expected = resolve_shard_dir(base, cache)
+ identity = provenance.BaseRevision(base)
+ with provenance.base_revision_scope(identity):
+ assert provenance.base_revision_kwargs(base) == {}
+ assert resolve_shard_dir(base, cache) == expected
+ assert identity.revision == SHA and identity.source == "cache_path"
+ moving_hub.assert_not_called()
+
+
[email protected]("mode", ["offline", "failed"])
+def test_unknown_stream_weights_keep_unpinned_download_and_cache(
+ tmp_path, monkeypatch, moving_hub, mode,
+):
+ from soup_cli.utils import hubs, spectrum_scan
+ from soup_cli.utils.layer_shard import resolve_shard_dir
+
+ if mode == "offline":
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ else:
+ moving_hub.side_effect = OSError("metadata unavailable")
+ snapshot = tmp_path / "snapshot"
+ snapshot.mkdir()
+ (snapshot / "model.safetensors").write_bytes(b"existing local weight fixture")
+ calls = []
+
+ def download(model_id, **kwargs):
+ assert model_id == BASE
+ assert "revision" not in kwargs
+ calls.append(kwargs)
+ return str(snapshot)
+
+ monkeypatch.setattr(hubs, "snapshot_download", download)
+ cache = str(tmp_path / "cache")
+ original_slot = resolve_shard_dir(BASE, cache)
+ identity = provenance.BaseRevision(BASE)
+ with provenance.base_revision_scope(identity):
+ assert spectrum_scan.resolve_model_weights(BASE) == str(snapshot)
+ assert resolve_shard_dir(BASE, cache) == original_slot
+ assert len(calls) == 1
+ assert identity.revision is None
+ assert identity.source == ("offline" if mode == "offline" else "resolution_failed")
+ assert moving_hub.call_count == (0 if mode == "offline" else 1)
diff --git a/tests/test_base_revision_loading.py b/tests/test_base_revision_loading.py
index 937303c6..75b93c95 100644
--- a/tests/test_base_revision_loading.py
+++ b/tests/test_base_revision_loading.py
@@ -225,7 +225,9 @@ def test_dry_run_does_not_capture_training_identity(tmp_path, monkeypatch):
capture.assert_not_called()
[email protected]("mode", ["bound", "offline", "distributed", "non-hf"])
+ "bound", "streaming", "unsloth", "offline", "distributed", "non-hf",
+])
@pytest.mark.parametrize("source_state", ["unchanged", "changed", "deleted"])
def test_train_cli_records_identity_and_writes_final_artifact(
tmp_path, monkeypatch, moving_hub, mode, source_state,
@@ -244,6 +246,7 @@ def test_train_cli_records_identity_and_writes_final_artifact(
monkeypatch.setattr("soup_cli.utils.gpu.detect_device", lambda: ("cpu", "CPU"))
monkeypatch.setattr("soup_cli.utils.gpu.get_gpu_info", lambda: {})
alternate_snapshot = None
+ bound = mode in {"bound", "streaming"}
def alternate_hub(hub, model_id, *, local_dir):
nonlocal alternate_snapshot
@@ -261,15 +264,17 @@ def test_train_cli_records_identity_and_writes_final_artifact(
'{"instruction":"hi", "output":"hello"}\n', encoding="utf-8",
)
(tmp_path / "soup.yaml").write_text(
- f"base: {BASE}\ndata: {{train: data.jsonl, format: alpaca, val_split: 0}}\n"
+ f"base: {BASE}\nbackend: {'unsloth' if mode == 'unsloth' else 'transformers'}\n"
+ "data: {train: data.jsonl, format: alpaca, val_split: 0}\n"
"training: {epochs: 1, quantization: none, batch_size: 1, "
+ f"stream_layers: {str(mode == 'streaming').lower()}, "
f"hub: {'modelscope' if mode == 'non-hf' else 'hf'}}}\noutput: final\n",
encoding="utf-8",
)
import hashlib
stored_hash = hashlib.sha256((tmp_path / "data.jsonl").read_bytes()).hexdigest()
- expected = {"revision": REVISION} if mode == "bound" else {}
+ expected = {"revision": REVISION} if bound else {}
class Trainer:
def setup(self, dataset):
@@ -303,15 +308,16 @@ def test_train_cli_records_identity_and_writes_final_artifact(
assert row["status"] == "completed"
assert row["base_model"] == BASE
assert json.loads(row["config_json"])["base"] == BASE
- assert row["base_revision"] == (REVISION if mode == "bound" else None)
+ assert row["base_revision"] == (REVISION if bound else None)
assert row["base_revision_source"] == {
- "bound": "hub_commit", "offline": "offline", "distributed": "unsupported_backend",
+ "bound": "hub_commit", "streaming": "hub_commit", "unsloth": "unsupported_backend",
+ "offline": "offline", "distributed": "unsupported_backend",
"non-hf": "unsupported_backend",
}[mode]
metadata = provenance.read_run_provenance(output)
assert metadata["run_id"] == row["run_id"]
assert metadata["base_revision"] == row["base_revision"]
assert metadata["training_data_sha256"] == stored_hash
- assert moving_hub.call_count == (1 if mode == "bound" else 0)
+ assert moving_hub.call_count == (1 if bound else 0)
assert provenance.base_revision_kwargs(BASE) == {}
tracker.close()