Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
feat(provenance): coordinate distributed base revisions (#2127)
Nurkhan Esenbeka7d31d68e1be
diff --git a/changelog.d/0.76.0/2127.changed.md b/changelog.d/0.76.0/2127.changed.mdnew file mode 100644index 00000000..0c924230--- /dev/null+++ b/changelog.d/0.76.0/2127.changed.md@@ -0,0 +1,9 @@+- Coordinate the base-model commit across distributed Hugging Face Transformers+ `soup train` workers: rank zero resolves once, broadcasts nullable identity before+ setup and compares rank receipts before recording final provenance. Offline+ resolution remains unknown; collective errors stop independent per-rank binding.+ This changes distributed Transformers runs from `unsupported_backend` to a bound+ identity with `revision=<sha>` on every rank when resolution succeeds. Initialize+ the shared group with Trainer's 1800-second collective timeout; malformed/stale+ distributed launch declarations fail instead of silently running unbound.+ This is a separate distributed slice of #1902 (#2127), not universal backend coverage.diff --git a/docs/backends-and-ops.md b/docs/backends-and-ops.mdindex 7403b6a8..e843ac54 100644--- a/docs/backends-and-ops.md+++ b/docs/backends-and-ops.md@@ -1005,7 +1005,23 @@ Loading a local adapter folder directly also reuses that commit when its`adapter_config.json` names the same base and has no conflicting explicit revision;the tokenizer still comes from the adapter folder.-**Remaining boundaries:** Non-HF hubs, Unsloth, MLX and distributed loads remain unbound and+For a multi-process Hugging Face Transformers `soup train` launch, global rank zero+resolves the base once and broadcasts its nullable result before trainer setup.+Soup initializes or reuses Accelerate's shared process group with Trainer's default+1800-second timeout. The first initialization sets the timeout for every collective,+including later Trainer operations; an existing group keeps its original timeout.+Offline/failed metadata requests+are broadcast as unknown, preserving existing loading behavior. A collective+failure stops the run rather than letting workers resolve independent moving heads.+Ranks compare identity receipts after setup and training; a conflicting loaded+model commit on any rank clears the bound revision for every rank before recording+the final artifact. CPU/Gloo coordination is covered by real two-process tests on+Linux; simulated-rank checks run on every platform. The real cases skip on macOS+and Windows because env:// rendezvous can time out on loaded macOS runners. Rank+failures include both child logs after cleanup. NCCL/FSDP hardware execution is+not established by those tests.++**Remaining boundaries:** Non-HF hubs, Unsloth and MLX loads remain unbound andare recorded as such; a cache-path hint on any backend is still unverified. Standalonewrapper/sweep invocations do not automatically establish a run scope. Metadata iswritten for the final artifact, not each intermediate checkpoint. Other exportersdiff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.pyindex 562d579a..8a30de9b 100644--- a/src/soup_cli/commands/train.py+++ b/src/soup_cli/commands/train.py@@ -1926,17 +1926,24 @@ def train(# to keep separate chains and sweep's copy fell ten tasks and the MLX# route behind. See soup_cli/trainer/dispatch.py.from soup_cli.trainer.dispatch import build_trainer- from soup_cli.utils.launcher import is_in_distributed- from soup_cli.utils.run_provenance import BaseRevision, base_revision_scope+ from soup_cli.utils.run_provenance import (+ BaseRevision,+ base_revision_scope,+ confirm_distributed_base_revision,+ synchronize_base_revision,+ )base_identity = BaseRevision(run_base,- binding_supported=(cfg.backend == "transformers" and hub_name == "hf"- and not is_in_distributed()),+ binding_supported=(cfg.backend == "transformers" and hub_name == "hf"),)+ synchronize_base_revision(base_identity, cpu=device == "cpu")with base_revision_scope(base_identity):trainer_wrapper = build_trainer(cfg, **trainer_kwargs)trainer_wrapper.setup(dataset)+ confirm_distributed_base_revision(+ base_identity, model=getattr(trainer_wrapper, "model", None),+ )tracker.record_base_revision(run_id, revision=base_identity.revision, source=base_identity.source,)@@ -2044,6 +2051,9 @@ def train(from soup_cli.utils.run_provenance import write_run_provenance+ confirm_distributed_base_revision(+ base_identity, model=getattr(trainer_wrapper, "model", None),+ )tracker.record_base_revision(run_id, revision=base_identity.revision, source=base_identity.source,)diff --git a/src/soup_cli/utils/run_provenance.py b/src/soup_cli/utils/run_provenance.pyindex 93b1b9e7..73bde9d0 100644--- a/src/soup_cli/utils/run_provenance.py+++ b/src/soup_cli/utils/run_provenance.py@@ -13,6 +13,7 @@ import tempfilefrom contextlib import contextmanagerfrom contextvars import ContextVarfrom dataclasses import dataclass, field+from datetime import timedeltafrom pathlib import Pathfrom typing import Any, Iterator@@ -22,6 +23,9 @@ PROVENANCE_FILENAME = "soup_run.json"_MAX_METADATA_BYTES = 64 * 1024_REVISION = re.compile(r"[0-9a-f]{40}")_SHA256 = re.compile(r"[0-9a-f]{64}")+# Match TrainingArguments.ddp_timeout. PartialState shares the first group's+# timeout with Trainer: it limits every collective, not only initialization.+_TRAINER_DDP_TIMEOUT_SECONDS = 1800_SOURCES = {"hub_commit", "cache_path", "local", "unknown", "unsupported_backend", "offline","resolution_failed",@@ -103,6 +107,63 @@ class BaseRevision:_ACTIVE_BASE: ContextVar[BaseRevision | None] = ContextVar("soup_base_revision", default=None)+def _distributed_state(identity: BaseRevision, *, cpu: bool = False) -> Any:+ """Initialize/reuse Trainer's Accelerate group only for a supported launch."""+ if not identity.binding_supported or int(os.environ.get("WORLD_SIZE", "1")) < 2:+ return None+ from accelerate import PartialState++ # Initialize/reuse Accelerate's shared group without shortening Trainer's+ # collective budget for dataset preparation, checkpoint saves or evaluation.+ state = PartialState(cpu=cpu, timeout=timedelta(seconds=_TRAINER_DDP_TIMEOUT_SECONDS))+ if state.num_processes < 2:+ raise RuntimeError("distributed base identity needs the launcher's process group")+ return state+++def synchronize_base_revision(identity: BaseRevision, *, cpu: bool = False) -> None:+ """Resolve once on global rank zero, broadcasting even an unknown outcome.++ Single-process resolution stays lazy. Provider errors/offline state preserve+ existing loader behavior; collective failure propagates instead of inventing+ independent bindings. The process group belongs to Accelerate/Trainer.+ """+ state = _distributed_state(identity, cpu=cpu)+ if state is None:+ return+ from accelerate.utils import broadcast_object_list++ payload: list[Any] = [None]+ if state.is_main_process:+ identity.load_kwargs(identity.base)+ payload[0] = identity.metadata("coordinator")+ broadcast_object_list(payload, from_process=0)+ recorded = _validate_metadata(payload[0])+ if recorded["base_model"] != identity.base:+ raise ValueError("distributed ranks disagree on the configured base model")+ identity.revision = recorded["base_revision"]+ identity.source = recorded["base_revision_source"]+ identity._pending = False+++def confirm_distributed_base_revision(identity: BaseRevision, *, model: Any = None) -> None:+ """Downgrade all ranks if a receiver contradicts the broadcast identity."""+ state = _distributed_state(identity)+ if state is None:+ return+ from accelerate.utils import gather_object++ loaded_revision = getattr(getattr(model, "config", None), "_commit_hash", None)+ if (identity.source == "hub_commit" and isinstance(loaded_revision, str)+ and loaded_revision != identity.revision):+ identity.revision, identity.source = None, "unsupported_backend"+ records = gather_object([identity.metadata("coordinator")])+ if not records:+ raise RuntimeError("distributed base identity received no rank receipts")+ if any(record != records[0] for record in records):+ identity.revision, identity.source, identity._pending = None, "unsupported_backend", False++@contextmanagerdef base_revision_scope(identity: BaseRevision) -> Iterator[None]:"""Keep one execution identity across setup/training; restore nested callers."""diff --git a/tests/test_base_revision_loading.py b/tests/test_base_revision_loading.pyindex 75b93c95..0b85ac07 100644--- a/tests/test_base_revision_loading.py+++ b/tests/test_base_revision_loading.py@@ -227,6 +227,7 @@ def test_dry_run_does_not_capture_training_identity(tmp_path, monkeypatch):@pytest.mark.parametrize("mode", ["bound", "streaming", "unsloth", "offline", "distributed", "non-hf",+ "distributed-setup", "distributed-late",])@pytest.mark.parametrize("source_state", ["unchanged", "changed", "deleted"])def test_train_cli_records_identity_and_writes_final_artifact(@@ -246,7 +247,17 @@ def test_train_cli_records_identity_and_writes_final_artifact(monkeypatch.setattr("soup_cli.utils.gpu.detect_device", lambda: ("cpu", "CPU"))monkeypatch.setattr("soup_cli.utils.gpu.get_gpu_info", lambda: {})alternate_snapshot = None- bound = mode in {"bound", "streaming"}+ bound = mode in {"bound", "streaming"} or mode.startswith("distributed")+ if mode.startswith("distributed"):+ import accelerate+ import accelerate.utils++ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setattr(accelerate, "PartialState", lambda **kwargs: SimpleNamespace(+ num_processes=2, is_main_process=True,+ ))+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list", lambda values, **kw: values)+ monkeypatch.setattr(accelerate.utils, "gather_object", lambda values: values * 2)def alternate_hub(hub, model_id, *, local_dir):nonlocal alternate_snapshot@@ -277,6 +288,8 @@ def test_train_cli_records_identity_and_writes_final_artifact(expected = {"revision": REVISION} if bound else {}class Trainer:+ model = SimpleNamespace(config=SimpleNamespace(_commit_hash=None))+def setup(self, dataset):assert dataset["train"]tracker = ExperimentTracker(tmp_path / "runs.db")@@ -285,9 +298,15 @@ def test_train_cli_records_identity_and_writes_final_artifact(finally:tracker.close()assert provenance.base_revision_kwargs(BASE) == expected+ if mode == "distributed-setup":+ self.model.config._commit_hash = "b" * 40def train(self, **kwargs):- assert provenance.base_revision_kwargs(BASE) == expected+ assert provenance.base_revision_kwargs(BASE) == (+ {} if mode == "distributed-setup" else expected+ )+ if mode == "distributed-late":+ self.model.config._commit_hash = "b" * 40if source_state == "changed":(tmp_path / "data.jsonl").write_text("later bytes", encoding="utf-8")elif source_state == "deleted":@@ -298,6 +317,8 @@ def test_train_cli_records_identity_and_writes_final_artifact(def build(cfg, **kwargs):assert cfg.base == (alternate_snapshot if mode == "non-hf" else BASE)+ if mode.startswith("distributed"):+ moving_hub.assert_called_once()return Trainer()monkeypatch.setattr("soup_cli.trainer.dispatch.build_trainer", build)@@ -308,10 +329,13 @@ def test_train_cli_records_identity_and_writes_final_artifact(assert row["status"] == "completed"assert row["base_model"] == BASEassert json.loads(row["config_json"])["base"] == BASE- assert row["base_revision"] == (REVISION if bound else None)+ assert row["base_revision"] == (+ REVISION if bound and mode not in {"distributed-setup", "distributed-late"} else None+ )assert row["base_revision_source"] == {"bound": "hub_commit", "streaming": "hub_commit", "unsloth": "unsupported_backend",- "offline": "offline", "distributed": "unsupported_backend",+ "offline": "offline", "distributed": "hub_commit",+ "distributed-setup": "unsupported_backend", "distributed-late": "unsupported_backend","non-hf": "unsupported_backend",}[mode]metadata = provenance.read_run_provenance(output)diff --git a/tests/test_distributed_base_revision.py b/tests/test_distributed_base_revision.pynew file mode 100644index 00000000..be783a85--- /dev/null+++ b/tests/test_distributed_base_revision.py@@ -0,0 +1,280 @@+"""Rank-zero base identity coordination, including real CPU process groups."""++import json+import os+import socket+import subprocess+import sys+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import Mock++import pytest++from soup_cli.utils import run_provenance as provenance++BASE = "owner/base"+SHA = "a" * 40+linux_only = pytest.mark.skipif(+ sys.platform != "linux",+ reason=(+ "real two-process CPU/Gloo checks run on Linux only; macOS env:// rendezvous "+ "can time out on loaded runners. Simulated-rank checks run on every platform."+ ),+)++[email protected]("rank", [0, 1])[email protected]("source", ["hub_commit", "offline", "resolution_failed"])+def test_each_rank_uses_only_the_coordinators_nullable_result(monkeypatch, rank, source):+ import accelerate+ import accelerate.utils++ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)+ if source == "offline":+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")+ probe = Mock(return_value=SimpleNamespace(sha=SHA))+ if source == "resolution_failed":+ probe.side_effect = OSError("private provider diagnostic")+ monkeypatch.setattr(provenance, "get_hf_api", lambda: SimpleNamespace(model_info=probe))+ # Two one-process nodes: each is its node's local main process, but only+ # global rank zero may resolve. A local-main check would resolve twice.+ state = SimpleNamespace(num_processes=2, is_main_process=rank == 0, is_local_main_process=True)+ factory = Mock(return_value=state)+ monkeypatch.setattr(accelerate, "PartialState", factory)+ payload = {"schema_version": 1, "run_id": "coordinator", "base_model": BASE,+ "base_revision": SHA if source == "hub_commit" else None,+ "base_revision_source": source, "training_data_sha256": None}++ def broadcast(objects, from_process):+ assert from_process == 0+ assert objects == ([payload] if rank == 0 else [None])+ objects[0] = payload++ send = Mock(side_effect=broadcast)+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list", send)+ identity = provenance.BaseRevision(BASE)+ provenance.synchronize_base_revision(identity, cpu=True)+ expected = {"revision": SHA} if source == "hub_commit" else {}+ assert identity.load_kwargs(BASE) == expected+ assert identity.load_kwargs(BASE) == expected+ assert identity.load_kwargs("other/teacher") == {}+ assert identity.source == source+ assert probe.call_count == (1 if rank == 0 and source != "offline" else 0)+ assert factory.call_args.kwargs["cpu"] is True+ assert factory.call_args.kwargs["timeout"].total_seconds() == 1800+ send.assert_called_once()++[email protected]("cpu", [False, True])+def test_initial_group_timeout_is_not_shorter_than_trainers_collective_default(monkeypatch, cpu):+ import accelerate+ from transformers import TrainingArguments++ monkeypatch.setenv("WORLD_SIZE", "2")+ factory = Mock(return_value=SimpleNamespace(num_processes=2))+ monkeypatch.setattr(accelerate, "PartialState", factory)+ provenance._distributed_state(provenance.BaseRevision(BASE), cpu=cpu)+ # PartialState shares the first group's timeout with Trainer. Compare with+ # Trainer's actual default, not the constant being tested in Soup.+ trainer_seconds = TrainingArguments.__dataclass_fields__["ddp_timeout"].default+ assert factory.call_args.kwargs["timeout"].total_seconds() >= trainer_seconds+ assert factory.call_args.kwargs["cpu"] is cpu+++def test_single_process_stays_lazy_and_unsupported_backends_stay_unbound(monkeypatch):+ monkeypatch.setenv("WORLD_SIZE", "1")+ probe = Mock(side_effect=AssertionError("unexpected metadata request"))+ monkeypatch.setattr(provenance, "get_hf_api", probe)+ identity = provenance.BaseRevision(BASE)+ provenance.synchronize_base_revision(identity)+ assert identity._pending is True+ unsupported = provenance.BaseRevision(BASE, binding_supported=False)+ provenance.synchronize_base_revision(unsupported)+ assert unsupported.load_kwargs(BASE) == {}+ probe.assert_not_called()++[email protected]("bad", [None, {"schema_version": 9}, {+ "schema_version": 1, "run_id": "coordinator", "base_model": "different/base",+ "base_revision": SHA, "base_revision_source": "hub_commit",+}])+def test_bad_broadcast_cannot_fall_back_to_independent_resolution(monkeypatch, bad):+ import accelerate+ import accelerate.utils++ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setattr(accelerate, "PartialState", lambda **kwargs: SimpleNamespace(+ num_processes=2, is_main_process=False,+ ))+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list",+ lambda values, **kwargs: values.__setitem__(0, bad))+ probe = Mock(side_effect=AssertionError("must not resolve on a worker"))+ monkeypatch.setattr(provenance, "get_hf_api", probe)+ with pytest.raises(ValueError):+ provenance.synchronize_base_revision(provenance.BaseRevision(BASE), cpu=True)+ probe.assert_not_called()+++def test_missing_process_group_does_not_claim_independent_bindings(monkeypatch):+ import accelerate++ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setattr(accelerate, "PartialState", lambda **kw: SimpleNamespace(num_processes=1))+ with pytest.raises(RuntimeError, match="process group"):+ provenance.synchronize_base_revision(provenance.BaseRevision(BASE), cpu=True)++[email protected]("lost", [False, True])+def test_a_lost_receipt_on_any_rank_downgrades_every_participant(monkeypatch, lost):+ import accelerate+ import accelerate.utils++ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setattr(accelerate, "PartialState", lambda **kw: SimpleNamespace(num_processes=2))+ identity = provenance.BaseRevision(BASE)+ identity.revision, identity.source, identity._pending = SHA, "hub_commit", False+ record = identity.metadata("coordinator")+ other = {**record, "base_revision": None, "base_revision_source": "unsupported_backend"}+ monkeypatch.setattr(accelerate.utils, "gather_object",+ lambda values: [*values, other if lost else record])+ provenance.confirm_distributed_base_revision(identity)+ assert identity.revision == (None if lost else SHA)+ assert identity.source == ("unsupported_backend" if lost else "hub_commit")+++def test_collective_failure_is_not_reinterpreted_as_offline(monkeypatch):+ import accelerate+ import accelerate.utils++ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setattr(accelerate, "PartialState", lambda **kw: SimpleNamespace(+ num_processes=2, is_main_process=False,+ ))+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list",+ Mock(side_effect=RuntimeError("collective failed")))+ with pytest.raises(RuntimeError, match="collective failed"):+ provenance.synchronize_base_revision(provenance.BaseRevision(BASE), cpu=True)+++WORKER = '''+import json+import os+import sys+from datetime import timedelta+from pathlib import Path+from types import SimpleNamespace+import torch.distributed as distributed+from soup_cli.utils import run_provenance as provenance++root, rank = Path(sys.argv[1]), int(sys.argv[2])+mode, initialize = sys.argv[3], sys.argv[4] == "True"+calls = []+def model_info(base, timeout):+ calls.append(base)+ if mode == "failed":+ raise OSError("secret should never be broadcast")+ return SimpleNamespace(sha=("a" if rank == 0 else "b") * 40)+provenance.get_hf_api = lambda: SimpleNamespace(model_info=model_info)+if mode == "offline" and rank == 0:+ os.environ["HF_HUB_OFFLINE"] = "1"+if initialize:+ distributed.init_process_group("gloo", init_method=(root / "rendezvous").as_uri(),+ rank=rank, world_size=2, timeout=timedelta(seconds=45))+try:+ identity = provenance.BaseRevision("owner/base")+ provenance.synchronize_base_revision(identity, cpu=True)+ with provenance.base_revision_scope(identity):+ receivers = [provenance.base_revision_kwargs("owner/base") for _ in range(3)]+ assert provenance.base_revision_kwargs("owner/teacher") == {}+ model = SimpleNamespace(config=SimpleNamespace(+ _commit_hash="b" * 40 if mode == "lost" and rank == 1 else None))+ provenance.confirm_distributed_base_revision(identity, model=model)+ provenance.write_run_provenance(root / f"artifact-{rank}", identity.metadata("run"))+ (root / f"rank-{rank}.json").write_text(json.dumps({+ "metadata": provenance.read_run_provenance(root / f"artifact-{rank}"),+ "receivers": receivers, "calls": calls,+ }), encoding="utf-8")+finally:+ if distributed.is_initialized():+ distributed.destroy_process_group()+'''+++def _wait_for_ranks(processes: list[subprocess.Popen], log_paths: list[Path]) -> None:+ """Reap every child before reporting a failed rank and both ranks' diagnostics."""+ failure = None+ try:+ for rank, process in enumerate(processes):+ try:+ returncode = process.wait(timeout=75)+ except subprocess.TimeoutExpired:+ failure = f"rank {rank} timed out after 75 seconds"+ break+ if returncode != 0:+ failure = f"rank {rank} exited with {returncode}"+ break+ finally:+ for process in processes:+ if process.poll() is None:+ process.kill()+ process.wait(timeout=10)+ if failure is not None:+ diagnostics = "\n".join(+ f"--- rank {rank} ---\n{path.read_text(encoding='utf-8', errors='replace')}"+ for rank, path in enumerate(log_paths)+ )+ pytest.fail(f"{failure}\n{diagnostics}", pytrace=False)+++@linux_only[email protected]("mode", ["bound", "offline", "failed", "lost"])[email protected]("initialize", [False, True])+def test_two_real_cpu_processes_share_identity_and_receipts(tmp_path, mode, initialize):+ distributed = pytest.importorskip("torch.distributed")+ if not distributed.is_gloo_available():+ pytest.skip("installed torch has no CPU Gloo backend")+ worker = tmp_path / "worker.py"+ worker.write_text(WORKER, encoding="utf-8")+ with socket.socket() as address:+ address.bind(("127.0.0.1", 0))+ port = address.getsockname()[1]+ processes, logs, log_paths = [], [], []+ try:+ for rank in range(2):+ env = os.environ.copy()+ env.update(RANK=str(rank), LOCAL_RANK=str(rank), WORLD_SIZE="2", LOCAL_WORLD_SIZE="2",+ MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port), ACCELERATE_USE_CPU="true",+ USE_LIBUV="0", OMP_NUM_THREADS="1")+ for name in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "ACCELERATE_USE_FSDP",+ "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_MIXED_PRECISION"):+ env.pop(name, None)+ log_path = tmp_path / f"rank-{rank}.log"+ log = log_path.open("w", encoding="utf-8")+ logs.append(log)+ log_paths.append(log_path)+ processes.append(subprocess.Popen(+ [sys.executable, "-u", str(worker), str(tmp_path),+ str(rank), mode, str(initialize)],+ env=env, stdout=log, stderr=subprocess.STDOUT,+ ))+ _wait_for_ranks(processes, log_paths)+ finally:+ for process in processes:+ if process.poll() is None:+ process.kill()+ process.wait(timeout=10)+ for log in logs:+ log.close()+ rows = [json.loads((tmp_path / f"rank-{rank}.json").read_text()) for rank in range(2)]+ assert rows[0]["metadata"] == rows[1]["metadata"]+ expected_source = {"bound": "hub_commit", "offline": "offline", "failed": "resolution_failed",+ "lost": "unsupported_backend"}[mode]+ assert rows[0]["metadata"]["base_revision_source"] == expected_source+ assert rows[0]["metadata"]["base_revision"] == (SHA if mode == "bound" else None)+ assert rows[0]["calls"] == ([] if mode == "offline" else [BASE])+ assert rows[1]["calls"] == []+ expected_kwargs = {"revision": SHA} if mode in {"bound", "lost"} else {}+ assert all(row["receivers"] == [expected_kwargs] * 3 for row in rows)diff --git a/tests/test_distributed_base_revision_harness.py b/tests/test_distributed_base_revision_harness.pynew file mode 100644index 00000000..fe1f7b16--- /dev/null+++ b/tests/test_distributed_base_revision_harness.py@@ -0,0 +1,68 @@+"""Platform selection and useful failures for the real two-rank Gloo checks."""++import runpy+import subprocess+import sys+from unittest.mock import Mock++import pytest++from tests import test_distributed_base_revision as ranks++[email protected]("platform,skipped", [("linux", False), ("darwin", True), ("win32", True)])+def test_only_real_gloo_cases_are_limited_to_linux(monkeypatch, platform, skipped):+ monkeypatch.setattr(sys, "platform", platform)+ namespace = runpy.run_path(ranks.__file__)+ real = namespace["test_two_real_cpu_processes_share_identity_and_receipts"]+ gates = [mark for mark in real.pytestmark if mark.name == "skipif"]+ assert len(gates) == 1+ assert gates[0].args == (skipped,)+ assert "Linux" in gates[0].kwargs["reason"]+ assert "Gloo" in gates[0].kwargs["reason"]+ assert not namespace.get("pytestmark"), "simulated ranks must run on every platform"+ for name, value in namespace.items():+ if name.startswith("test_") and value is not real:+ assert not any(mark.name in {"skip", "skipif"}+ for mark in getattr(value, "pytestmark", ())), name++[email protected]("rank", [0, 1])[email protected]("failure", ["timeout", "exit"])+def test_rank_failure_reports_both_logs_and_reaps_the_children(tmp_path, rank, failure):+ logs = [tmp_path / f"rank-{index}.log" for index in range(2)]+ for index, path in enumerate(logs):+ path.write_text(f"rank {index} diagnostic", encoding="utf-8")+ processes = [Mock(), Mock()]+ for index, process in enumerate(processes):+ process.poll.return_value = 0 if index < rank else None+ process.wait.return_value = 0+ if failure == "timeout":+ processes[rank].wait.side_effect = [subprocess.TimeoutExpired("worker", 75), -9]+ else:+ processes[rank].wait.return_value = 7+ processes[rank].poll.return_value = 7+ with pytest.raises(pytest.fail.Exception) as error:+ ranks._wait_for_ranks(processes, logs)+ message = str(error.value)+ assert f"rank {rank}" in message+ assert ("timed out" if failure == "timeout" else "exited with 7") in message+ for index in range(2):+ assert f"rank {index} diagnostic" in message+ process = processes[index]+ if process.poll.return_value is None:+ process.kill.assert_called_once_with()+ assert process.wait.call_args.kwargs == {"timeout": 10}+ else:+ process.kill.assert_not_called()+++def test_successful_ranks_are_waited_for_without_being_killed(tmp_path):+ processes = [Mock(), Mock()]+ for process in processes:+ process.wait.return_value = process.poll.return_value = 0+ # Successful children need no diagnostic reads.+ ranks._wait_for_ranks(processes, [tmp_path / "absent-0", tmp_path / "absent-1"])+ for process in processes:+ process.wait.assert_called_once_with(timeout=75)+ process.kill.assert_not_called()