Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

feat(provenance): coordinate distributed base revisions (#2127)

Nurkhan Esenbeka7d31d68e1be
diff --git a/changelog.d/0.76.0/2127.changed.md b/changelog.d/0.76.0/2127.changed.md
new file mode 100644
index 00000000..0c924230
--- /dev/null
+++ b/changelog.d/0.76.0/2127.changed.md
@@ -0,0 +1,9 @@
+- Coordinate the base-model commit across distributed Hugging Face Transformers
+ `soup train` workers: rank zero resolves once, broadcasts nullable identity before
+ setup and compares rank receipts before recording final provenance. Offline
+ resolution remains unknown; collective errors stop independent per-rank binding.
+ This changes distributed Transformers runs from `unsupported_backend` to a bound
+ identity with `revision=<sha>` on every rank when resolution succeeds. Initialize
+ the shared group with Trainer's 1800-second collective timeout; malformed/stale
+ distributed launch declarations fail instead of silently running unbound.
+ This is a separate distributed slice of #1902 (#2127), not universal backend coverage.
diff --git a/docs/backends-and-ops.md b/docs/backends-and-ops.md
index 7403b6a8..e843ac54 100644
--- a/docs/backends-and-ops.md
+++ b/docs/backends-and-ops.md
@@ -1005,7 +1005,23 @@ Loading a local adapter folder directly also reuses that commit when its
`adapter_config.json` names the same base and has no conflicting explicit revision;
the tokenizer still comes from the adapter folder.
-**Remaining boundaries:** Non-HF hubs, Unsloth, MLX and distributed loads remain unbound and
+For a multi-process Hugging Face Transformers `soup train` launch, global rank zero
+resolves the base once and broadcasts its nullable result before trainer setup.
+Soup initializes or reuses Accelerate's shared process group with Trainer's default
+1800-second timeout. The first initialization sets the timeout for every collective,
+including later Trainer operations; an existing group keeps its original timeout.
+Offline/failed metadata requests
+are broadcast as unknown, preserving existing loading behavior. A collective
+failure stops the run rather than letting workers resolve independent moving heads.
+Ranks compare identity receipts after setup and training; a conflicting loaded
+model commit on any rank clears the bound revision for every rank before recording
+the final artifact. CPU/Gloo coordination is covered by real two-process tests on
+Linux; simulated-rank checks run on every platform. The real cases skip on macOS
+and Windows because env:// rendezvous can time out on loaded macOS runners. Rank
+failures include both child logs after cleanup. NCCL/FSDP hardware execution is
+not established by those tests.
+
+**Remaining boundaries:** Non-HF hubs, Unsloth and MLX loads remain unbound and
are recorded as such; a cache-path hint on any backend is still unverified. Standalone
wrapper/sweep invocations do not automatically establish a run scope. Metadata is
written for the final artifact, not each intermediate checkpoint. Other exporters
diff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.py
index 562d579a..8a30de9b 100644
--- a/src/soup_cli/commands/train.py
+++ b/src/soup_cli/commands/train.py
@@ -1926,17 +1926,24 @@ def train(
# to keep separate chains and sweep's copy fell ten tasks and the MLX
# route behind. See soup_cli/trainer/dispatch.py.
from soup_cli.trainer.dispatch import build_trainer
- from soup_cli.utils.launcher import is_in_distributed
- from soup_cli.utils.run_provenance import BaseRevision, base_revision_scope
+ from soup_cli.utils.run_provenance import (
+ BaseRevision,
+ base_revision_scope,
+ confirm_distributed_base_revision,
+ synchronize_base_revision,
+ )
base_identity = BaseRevision(
run_base,
- binding_supported=(cfg.backend == "transformers" and hub_name == "hf"
- and not is_in_distributed()),
+ binding_supported=(cfg.backend == "transformers" and hub_name == "hf"),
)
+ synchronize_base_revision(base_identity, cpu=device == "cpu")
with base_revision_scope(base_identity):
trainer_wrapper = build_trainer(cfg, **trainer_kwargs)
trainer_wrapper.setup(dataset)
+ confirm_distributed_base_revision(
+ base_identity, model=getattr(trainer_wrapper, "model", None),
+ )
tracker.record_base_revision(
run_id, revision=base_identity.revision, source=base_identity.source,
)
@@ -2044,6 +2051,9 @@ def train(
from soup_cli.utils.run_provenance import write_run_provenance
+ confirm_distributed_base_revision(
+ base_identity, model=getattr(trainer_wrapper, "model", None),
+ )
tracker.record_base_revision(
run_id, revision=base_identity.revision, source=base_identity.source,
)
diff --git a/src/soup_cli/utils/run_provenance.py b/src/soup_cli/utils/run_provenance.py
index 93b1b9e7..73bde9d0 100644
--- a/src/soup_cli/utils/run_provenance.py
+++ b/src/soup_cli/utils/run_provenance.py
@@ -13,6 +13,7 @@ import tempfile
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
+from datetime import timedelta
from pathlib import Path
from typing import Any, Iterator
@@ -22,6 +23,9 @@ PROVENANCE_FILENAME = "soup_run.json"
_MAX_METADATA_BYTES = 64 * 1024
_REVISION = re.compile(r"[0-9a-f]{40}")
_SHA256 = re.compile(r"[0-9a-f]{64}")
+# Match TrainingArguments.ddp_timeout. PartialState shares the first group's
+# timeout with Trainer: it limits every collective, not only initialization.
+_TRAINER_DDP_TIMEOUT_SECONDS = 1800
_SOURCES = {
"hub_commit", "cache_path", "local", "unknown", "unsupported_backend", "offline",
"resolution_failed",
@@ -103,6 +107,63 @@ class BaseRevision:
_ACTIVE_BASE: ContextVar[BaseRevision | None] = ContextVar("soup_base_revision", default=None)
+def _distributed_state(identity: BaseRevision, *, cpu: bool = False) -> Any:
+ """Initialize/reuse Trainer's Accelerate group only for a supported launch."""
+ if not identity.binding_supported or int(os.environ.get("WORLD_SIZE", "1")) < 2:
+ return None
+ from accelerate import PartialState
+
+ # Initialize/reuse Accelerate's shared group without shortening Trainer's
+ # collective budget for dataset preparation, checkpoint saves or evaluation.
+ state = PartialState(cpu=cpu, timeout=timedelta(seconds=_TRAINER_DDP_TIMEOUT_SECONDS))
+ if state.num_processes < 2:
+ raise RuntimeError("distributed base identity needs the launcher's process group")
+ return state
+
+
+def synchronize_base_revision(identity: BaseRevision, *, cpu: bool = False) -> None:
+ """Resolve once on global rank zero, broadcasting even an unknown outcome.
+
+ Single-process resolution stays lazy. Provider errors/offline state preserve
+ existing loader behavior; collective failure propagates instead of inventing
+ independent bindings. The process group belongs to Accelerate/Trainer.
+ """
+ state = _distributed_state(identity, cpu=cpu)
+ if state is None:
+ return
+ from accelerate.utils import broadcast_object_list
+
+ payload: list[Any] = [None]
+ if state.is_main_process:
+ identity.load_kwargs(identity.base)
+ payload[0] = identity.metadata("coordinator")
+ broadcast_object_list(payload, from_process=0)
+ recorded = _validate_metadata(payload[0])
+ if recorded["base_model"] != identity.base:
+ raise ValueError("distributed ranks disagree on the configured base model")
+ identity.revision = recorded["base_revision"]
+ identity.source = recorded["base_revision_source"]
+ identity._pending = False
+
+
+def confirm_distributed_base_revision(identity: BaseRevision, *, model: Any = None) -> None:
+ """Downgrade all ranks if a receiver contradicts the broadcast identity."""
+ state = _distributed_state(identity)
+ if state is None:
+ return
+ from accelerate.utils import gather_object
+
+ loaded_revision = getattr(getattr(model, "config", None), "_commit_hash", None)
+ if (identity.source == "hub_commit" and isinstance(loaded_revision, str)
+ and loaded_revision != identity.revision):
+ identity.revision, identity.source = None, "unsupported_backend"
+ records = gather_object([identity.metadata("coordinator")])
+ if not records:
+ raise RuntimeError("distributed base identity received no rank receipts")
+ if any(record != records[0] for record in records):
+ identity.revision, identity.source, identity._pending = None, "unsupported_backend", False
+
+
@contextmanager
def base_revision_scope(identity: BaseRevision) -> Iterator[None]:
"""Keep one execution identity across setup/training; restore nested callers."""
diff --git a/tests/test_base_revision_loading.py b/tests/test_base_revision_loading.py
index 75b93c95..0b85ac07 100644
--- a/tests/test_base_revision_loading.py
+++ b/tests/test_base_revision_loading.py
@@ -227,6 +227,7 @@ def test_dry_run_does_not_capture_training_identity(tmp_path, monkeypatch):
@pytest.mark.parametrize("mode", [
"bound", "streaming", "unsloth", "offline", "distributed", "non-hf",
+ "distributed-setup", "distributed-late",
])
@pytest.mark.parametrize("source_state", ["unchanged", "changed", "deleted"])
def test_train_cli_records_identity_and_writes_final_artifact(
@@ -246,7 +247,17 @@ def test_train_cli_records_identity_and_writes_final_artifact(
monkeypatch.setattr("soup_cli.utils.gpu.detect_device", lambda: ("cpu", "CPU"))
monkeypatch.setattr("soup_cli.utils.gpu.get_gpu_info", lambda: {})
alternate_snapshot = None
- bound = mode in {"bound", "streaming"}
+ bound = mode in {"bound", "streaming"} or mode.startswith("distributed")
+ if mode.startswith("distributed"):
+ import accelerate
+ import accelerate.utils
+
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setattr(accelerate, "PartialState", lambda **kwargs: SimpleNamespace(
+ num_processes=2, is_main_process=True,
+ ))
+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list", lambda values, **kw: values)
+ monkeypatch.setattr(accelerate.utils, "gather_object", lambda values: values * 2)
def alternate_hub(hub, model_id, *, local_dir):
nonlocal alternate_snapshot
@@ -277,6 +288,8 @@ def test_train_cli_records_identity_and_writes_final_artifact(
expected = {"revision": REVISION} if bound else {}
class Trainer:
+ model = SimpleNamespace(config=SimpleNamespace(_commit_hash=None))
+
def setup(self, dataset):
assert dataset["train"]
tracker = ExperimentTracker(tmp_path / "runs.db")
@@ -285,9 +298,15 @@ def test_train_cli_records_identity_and_writes_final_artifact(
finally:
tracker.close()
assert provenance.base_revision_kwargs(BASE) == expected
+ if mode == "distributed-setup":
+ self.model.config._commit_hash = "b" * 40
def train(self, **kwargs):
- assert provenance.base_revision_kwargs(BASE) == expected
+ assert provenance.base_revision_kwargs(BASE) == (
+ {} if mode == "distributed-setup" else expected
+ )
+ if mode == "distributed-late":
+ self.model.config._commit_hash = "b" * 40
if source_state == "changed":
(tmp_path / "data.jsonl").write_text("later bytes", encoding="utf-8")
elif source_state == "deleted":
@@ -298,6 +317,8 @@ def test_train_cli_records_identity_and_writes_final_artifact(
def build(cfg, **kwargs):
assert cfg.base == (alternate_snapshot if mode == "non-hf" else BASE)
+ if mode.startswith("distributed"):
+ moving_hub.assert_called_once()
return Trainer()
monkeypatch.setattr("soup_cli.trainer.dispatch.build_trainer", build)
@@ -308,10 +329,13 @@ def test_train_cli_records_identity_and_writes_final_artifact(
assert row["status"] == "completed"
assert row["base_model"] == BASE
assert json.loads(row["config_json"])["base"] == BASE
- assert row["base_revision"] == (REVISION if bound else None)
+ assert row["base_revision"] == (
+ REVISION if bound and mode not in {"distributed-setup", "distributed-late"} else None
+ )
assert row["base_revision_source"] == {
"bound": "hub_commit", "streaming": "hub_commit", "unsloth": "unsupported_backend",
- "offline": "offline", "distributed": "unsupported_backend",
+ "offline": "offline", "distributed": "hub_commit",
+ "distributed-setup": "unsupported_backend", "distributed-late": "unsupported_backend",
"non-hf": "unsupported_backend",
}[mode]
metadata = provenance.read_run_provenance(output)
diff --git a/tests/test_distributed_base_revision.py b/tests/test_distributed_base_revision.py
new file mode 100644
index 00000000..be783a85
--- /dev/null
+++ b/tests/test_distributed_base_revision.py
@@ -0,0 +1,280 @@
+"""Rank-zero base identity coordination, including real CPU process groups."""
+
+import json
+import os
+import socket
+import subprocess
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+from soup_cli.utils import run_provenance as provenance
+
+BASE = "owner/base"
+SHA = "a" * 40
+linux_only = pytest.mark.skipif(
+ sys.platform != "linux",
+ reason=(
+ "real two-process CPU/Gloo checks run on Linux only; macOS env:// rendezvous "
+ "can time out on loaded runners. Simulated-rank checks run on every platform."
+ ),
+)
+
+
[email protected]("rank", [0, 1])
[email protected]("source", ["hub_commit", "offline", "resolution_failed"])
+def test_each_rank_uses_only_the_coordinators_nullable_result(monkeypatch, rank, source):
+ import accelerate
+ import accelerate.utils
+
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)
+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)
+ if source == "offline":
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ probe = Mock(return_value=SimpleNamespace(sha=SHA))
+ if source == "resolution_failed":
+ probe.side_effect = OSError("private provider diagnostic")
+ monkeypatch.setattr(provenance, "get_hf_api", lambda: SimpleNamespace(model_info=probe))
+ # Two one-process nodes: each is its node's local main process, but only
+ # global rank zero may resolve. A local-main check would resolve twice.
+ state = SimpleNamespace(num_processes=2, is_main_process=rank == 0, is_local_main_process=True)
+ factory = Mock(return_value=state)
+ monkeypatch.setattr(accelerate, "PartialState", factory)
+ payload = {"schema_version": 1, "run_id": "coordinator", "base_model": BASE,
+ "base_revision": SHA if source == "hub_commit" else None,
+ "base_revision_source": source, "training_data_sha256": None}
+
+ def broadcast(objects, from_process):
+ assert from_process == 0
+ assert objects == ([payload] if rank == 0 else [None])
+ objects[0] = payload
+
+ send = Mock(side_effect=broadcast)
+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list", send)
+ identity = provenance.BaseRevision(BASE)
+ provenance.synchronize_base_revision(identity, cpu=True)
+ expected = {"revision": SHA} if source == "hub_commit" else {}
+ assert identity.load_kwargs(BASE) == expected
+ assert identity.load_kwargs(BASE) == expected
+ assert identity.load_kwargs("other/teacher") == {}
+ assert identity.source == source
+ assert probe.call_count == (1 if rank == 0 and source != "offline" else 0)
+ assert factory.call_args.kwargs["cpu"] is True
+ assert factory.call_args.kwargs["timeout"].total_seconds() == 1800
+ send.assert_called_once()
+
+
[email protected]("cpu", [False, True])
+def test_initial_group_timeout_is_not_shorter_than_trainers_collective_default(monkeypatch, cpu):
+ import accelerate
+ from transformers import TrainingArguments
+
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ factory = Mock(return_value=SimpleNamespace(num_processes=2))
+ monkeypatch.setattr(accelerate, "PartialState", factory)
+ provenance._distributed_state(provenance.BaseRevision(BASE), cpu=cpu)
+ # PartialState shares the first group's timeout with Trainer. Compare with
+ # Trainer's actual default, not the constant being tested in Soup.
+ trainer_seconds = TrainingArguments.__dataclass_fields__["ddp_timeout"].default
+ assert factory.call_args.kwargs["timeout"].total_seconds() >= trainer_seconds
+ assert factory.call_args.kwargs["cpu"] is cpu
+
+
+def test_single_process_stays_lazy_and_unsupported_backends_stay_unbound(monkeypatch):
+ monkeypatch.setenv("WORLD_SIZE", "1")
+ probe = Mock(side_effect=AssertionError("unexpected metadata request"))
+ monkeypatch.setattr(provenance, "get_hf_api", probe)
+ identity = provenance.BaseRevision(BASE)
+ provenance.synchronize_base_revision(identity)
+ assert identity._pending is True
+ unsupported = provenance.BaseRevision(BASE, binding_supported=False)
+ provenance.synchronize_base_revision(unsupported)
+ assert unsupported.load_kwargs(BASE) == {}
+ probe.assert_not_called()
+
+
[email protected]("bad", [None, {"schema_version": 9}, {
+ "schema_version": 1, "run_id": "coordinator", "base_model": "different/base",
+ "base_revision": SHA, "base_revision_source": "hub_commit",
+}])
+def test_bad_broadcast_cannot_fall_back_to_independent_resolution(monkeypatch, bad):
+ import accelerate
+ import accelerate.utils
+
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setattr(accelerate, "PartialState", lambda **kwargs: SimpleNamespace(
+ num_processes=2, is_main_process=False,
+ ))
+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list",
+ lambda values, **kwargs: values.__setitem__(0, bad))
+ probe = Mock(side_effect=AssertionError("must not resolve on a worker"))
+ monkeypatch.setattr(provenance, "get_hf_api", probe)
+ with pytest.raises(ValueError):
+ provenance.synchronize_base_revision(provenance.BaseRevision(BASE), cpu=True)
+ probe.assert_not_called()
+
+
+def test_missing_process_group_does_not_claim_independent_bindings(monkeypatch):
+ import accelerate
+
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setattr(accelerate, "PartialState", lambda **kw: SimpleNamespace(num_processes=1))
+ with pytest.raises(RuntimeError, match="process group"):
+ provenance.synchronize_base_revision(provenance.BaseRevision(BASE), cpu=True)
+
+
[email protected]("lost", [False, True])
+def test_a_lost_receipt_on_any_rank_downgrades_every_participant(monkeypatch, lost):
+ import accelerate
+ import accelerate.utils
+
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setattr(accelerate, "PartialState", lambda **kw: SimpleNamespace(num_processes=2))
+ identity = provenance.BaseRevision(BASE)
+ identity.revision, identity.source, identity._pending = SHA, "hub_commit", False
+ record = identity.metadata("coordinator")
+ other = {**record, "base_revision": None, "base_revision_source": "unsupported_backend"}
+ monkeypatch.setattr(accelerate.utils, "gather_object",
+ lambda values: [*values, other if lost else record])
+ provenance.confirm_distributed_base_revision(identity)
+ assert identity.revision == (None if lost else SHA)
+ assert identity.source == ("unsupported_backend" if lost else "hub_commit")
+
+
+def test_collective_failure_is_not_reinterpreted_as_offline(monkeypatch):
+ import accelerate
+ import accelerate.utils
+
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setattr(accelerate, "PartialState", lambda **kw: SimpleNamespace(
+ num_processes=2, is_main_process=False,
+ ))
+ monkeypatch.setattr(accelerate.utils, "broadcast_object_list",
+ Mock(side_effect=RuntimeError("collective failed")))
+ with pytest.raises(RuntimeError, match="collective failed"):
+ provenance.synchronize_base_revision(provenance.BaseRevision(BASE), cpu=True)
+
+
+WORKER = '''
+import json
+import os
+import sys
+from datetime import timedelta
+from pathlib import Path
+from types import SimpleNamespace
+import torch.distributed as distributed
+from soup_cli.utils import run_provenance as provenance
+
+root, rank = Path(sys.argv[1]), int(sys.argv[2])
+mode, initialize = sys.argv[3], sys.argv[4] == "True"
+calls = []
+def model_info(base, timeout):
+ calls.append(base)
+ if mode == "failed":
+ raise OSError("secret should never be broadcast")
+ return SimpleNamespace(sha=("a" if rank == 0 else "b") * 40)
+provenance.get_hf_api = lambda: SimpleNamespace(model_info=model_info)
+if mode == "offline" and rank == 0:
+ os.environ["HF_HUB_OFFLINE"] = "1"
+if initialize:
+ distributed.init_process_group("gloo", init_method=(root / "rendezvous").as_uri(),
+ rank=rank, world_size=2, timeout=timedelta(seconds=45))
+try:
+ identity = provenance.BaseRevision("owner/base")
+ provenance.synchronize_base_revision(identity, cpu=True)
+ with provenance.base_revision_scope(identity):
+ receivers = [provenance.base_revision_kwargs("owner/base") for _ in range(3)]
+ assert provenance.base_revision_kwargs("owner/teacher") == {}
+ model = SimpleNamespace(config=SimpleNamespace(
+ _commit_hash="b" * 40 if mode == "lost" and rank == 1 else None))
+ provenance.confirm_distributed_base_revision(identity, model=model)
+ provenance.write_run_provenance(root / f"artifact-{rank}", identity.metadata("run"))
+ (root / f"rank-{rank}.json").write_text(json.dumps({
+ "metadata": provenance.read_run_provenance(root / f"artifact-{rank}"),
+ "receivers": receivers, "calls": calls,
+ }), encoding="utf-8")
+finally:
+ if distributed.is_initialized():
+ distributed.destroy_process_group()
+'''
+
+
+def _wait_for_ranks(processes: list[subprocess.Popen], log_paths: list[Path]) -> None:
+ """Reap every child before reporting a failed rank and both ranks' diagnostics."""
+ failure = None
+ try:
+ for rank, process in enumerate(processes):
+ try:
+ returncode = process.wait(timeout=75)
+ except subprocess.TimeoutExpired:
+ failure = f"rank {rank} timed out after 75 seconds"
+ break
+ if returncode != 0:
+ failure = f"rank {rank} exited with {returncode}"
+ break
+ finally:
+ for process in processes:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=10)
+ if failure is not None:
+ diagnostics = "\n".join(
+ f"--- rank {rank} ---\n{path.read_text(encoding='utf-8', errors='replace')}"
+ for rank, path in enumerate(log_paths)
+ )
+ pytest.fail(f"{failure}\n{diagnostics}", pytrace=False)
+
+
+@linux_only
[email protected]("mode", ["bound", "offline", "failed", "lost"])
[email protected]("initialize", [False, True])
+def test_two_real_cpu_processes_share_identity_and_receipts(tmp_path, mode, initialize):
+ distributed = pytest.importorskip("torch.distributed")
+ if not distributed.is_gloo_available():
+ pytest.skip("installed torch has no CPU Gloo backend")
+ worker = tmp_path / "worker.py"
+ worker.write_text(WORKER, encoding="utf-8")
+ with socket.socket() as address:
+ address.bind(("127.0.0.1", 0))
+ port = address.getsockname()[1]
+ processes, logs, log_paths = [], [], []
+ try:
+ for rank in range(2):
+ env = os.environ.copy()
+ env.update(RANK=str(rank), LOCAL_RANK=str(rank), WORLD_SIZE="2", LOCAL_WORLD_SIZE="2",
+ MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port), ACCELERATE_USE_CPU="true",
+ USE_LIBUV="0", OMP_NUM_THREADS="1")
+ for name in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "ACCELERATE_USE_FSDP",
+ "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_MIXED_PRECISION"):
+ env.pop(name, None)
+ log_path = tmp_path / f"rank-{rank}.log"
+ log = log_path.open("w", encoding="utf-8")
+ logs.append(log)
+ log_paths.append(log_path)
+ processes.append(subprocess.Popen(
+ [sys.executable, "-u", str(worker), str(tmp_path),
+ str(rank), mode, str(initialize)],
+ env=env, stdout=log, stderr=subprocess.STDOUT,
+ ))
+ _wait_for_ranks(processes, log_paths)
+ finally:
+ for process in processes:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=10)
+ for log in logs:
+ log.close()
+ rows = [json.loads((tmp_path / f"rank-{rank}.json").read_text()) for rank in range(2)]
+ assert rows[0]["metadata"] == rows[1]["metadata"]
+ expected_source = {"bound": "hub_commit", "offline": "offline", "failed": "resolution_failed",
+ "lost": "unsupported_backend"}[mode]
+ assert rows[0]["metadata"]["base_revision_source"] == expected_source
+ assert rows[0]["metadata"]["base_revision"] == (SHA if mode == "bound" else None)
+ assert rows[0]["calls"] == ([] if mode == "offline" else [BASE])
+ assert rows[1]["calls"] == []
+ expected_kwargs = {"revision": SHA} if mode in {"bound", "lost"} else {}
+ assert all(row["receivers"] == [expected_kwargs] * 3 for row in rows)
diff --git a/tests/test_distributed_base_revision_harness.py b/tests/test_distributed_base_revision_harness.py
new file mode 100644
index 00000000..fe1f7b16
--- /dev/null
+++ b/tests/test_distributed_base_revision_harness.py
@@ -0,0 +1,68 @@
+"""Platform selection and useful failures for the real two-rank Gloo checks."""
+
+import runpy
+import subprocess
+import sys
+from unittest.mock import Mock
+
+import pytest
+
+from tests import test_distributed_base_revision as ranks
+
+
[email protected]("platform,skipped", [("linux", False), ("darwin", True), ("win32", True)])
+def test_only_real_gloo_cases_are_limited_to_linux(monkeypatch, platform, skipped):
+ monkeypatch.setattr(sys, "platform", platform)
+ namespace = runpy.run_path(ranks.__file__)
+ real = namespace["test_two_real_cpu_processes_share_identity_and_receipts"]
+ gates = [mark for mark in real.pytestmark if mark.name == "skipif"]
+ assert len(gates) == 1
+ assert gates[0].args == (skipped,)
+ assert "Linux" in gates[0].kwargs["reason"]
+ assert "Gloo" in gates[0].kwargs["reason"]
+ assert not namespace.get("pytestmark"), "simulated ranks must run on every platform"
+ for name, value in namespace.items():
+ if name.startswith("test_") and value is not real:
+ assert not any(mark.name in {"skip", "skipif"}
+ for mark in getattr(value, "pytestmark", ())), name
+
+
[email protected]("rank", [0, 1])
[email protected]("failure", ["timeout", "exit"])
+def test_rank_failure_reports_both_logs_and_reaps_the_children(tmp_path, rank, failure):
+ logs = [tmp_path / f"rank-{index}.log" for index in range(2)]
+ for index, path in enumerate(logs):
+ path.write_text(f"rank {index} diagnostic", encoding="utf-8")
+ processes = [Mock(), Mock()]
+ for index, process in enumerate(processes):
+ process.poll.return_value = 0 if index < rank else None
+ process.wait.return_value = 0
+ if failure == "timeout":
+ processes[rank].wait.side_effect = [subprocess.TimeoutExpired("worker", 75), -9]
+ else:
+ processes[rank].wait.return_value = 7
+ processes[rank].poll.return_value = 7
+ with pytest.raises(pytest.fail.Exception) as error:
+ ranks._wait_for_ranks(processes, logs)
+ message = str(error.value)
+ assert f"rank {rank}" in message
+ assert ("timed out" if failure == "timeout" else "exited with 7") in message
+ for index in range(2):
+ assert f"rank {index} diagnostic" in message
+ process = processes[index]
+ if process.poll.return_value is None:
+ process.kill.assert_called_once_with()
+ assert process.wait.call_args.kwargs == {"timeout": 10}
+ else:
+ process.kill.assert_not_called()
+
+
+def test_successful_ranks_are_waited_for_without_being_killed(tmp_path):
+ processes = [Mock(), Mock()]
+ for process in processes:
+ process.wait.return_value = process.poll.return_value = 0
+ # Successful children need no diagnostic reads.
+ ranks._wait_for_ranks(processes, [tmp_path / "absent-0", tmp_path / "absent-1"])
+ for process in processes:
+ process.wait.assert_called_once_with(timeout=75)
+ process.kill.assert_not_called()