Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(doctor): validate config before printing environment success summary (#2100) (#2106)

Nariman Abdikarim7a6edd9b61a0
diff --git a/changelog.d/0.76.0/2106.fixed.md b/changelog.d/0.76.0/2106.fixed.md
new file mode 100644
index 00000000..a95b6178
--- /dev/null
+++ b/changelog.d/0.76.0/2106.fixed.md
@@ -0,0 +1 @@
+- Run config check before environment summary in `soup doctor` so a refused or invalid config does not falsely print "All checks passed!" before terminating with exit 2 (#2100, #2106 by @Nari-Ab).
diff --git a/src/soup_cli/commands/doctor.py b/src/soup_cli/commands/doctor.py
index 836b4c7b..9fa66063 100644
--- a/src/soup_cli/commands/doctor.py
+++ b/src/soup_cli/commands/doctor.py
@@ -327,14 +327,22 @@ def doctor(
console.print(
f"[dim] pip install {escape(' '.join(fix_parts))}[/]", highlight=False
)
- else:
+
+ loaded_cfg = None
+ if config:
+ # Load and validate the config before the environment summary (#2100).
+ # If the config is refused or cannot be loaded, doctor terminates with exit 2
+ # before asserting that "All checks passed!".
+ loaded_cfg = _load_doctor_config(config)
+
+ if not issues:
console.print("\n[bold green]All checks passed![/] Your environment is ready.")
# After the environment summary on purpose: "All checks passed" reports on
# the environment, and printing config findings above it read as though the
# green line covered them too.
- if config:
- _check_config_support(config)
+ if loaded_cfg is not None:
+ _report_config_support(loaded_cfg)
console.print(f"\n[dim]GitHub: [link={GITHUB_URL}]{GITHUB_URL}[/link][/]")
@@ -360,25 +368,12 @@ def _installed_version_str(import_name: str, pkg_name: str) -> str | None:
return str(version)
-def _check_config_support(config_path: str) -> None:
- """#755 — report the settings this config sets that its backend never reads.
-
- Only fields the user wrote and switched on are listed (a ``false`` or unset
- value is not; ``seed: 0`` is). A wall of 275 rows is not a pre-flight check,
- and the fields sitting at their schema default or written in their off
- position are not what anyone came here to ask about.
- """
- from soup_cli.config.backend_support import (
- DEFAULT_BACKEND,
- DEFAULT_MODALITY,
- check_config,
- unsupported_for,
- )
-
+def _load_doctor_config(config_path: str):
+ """Load config for doctor, exiting with code 2 on refusal or load error (#2100)."""
try:
from soup_cli.config.loader import load_config
- cfg = load_config(config_path)
+ return load_config(config_path)
except FileNotFoundError as exc:
console.print(f"\n[red]Config not found:[/] {config_path}")
# Non-zero deliberately: this leg is meant to be gate-able in CI, and a
@@ -397,6 +392,22 @@ def _check_config_support(config_path: str) -> None:
console.print(f"\n[red]Config could not be loaded:[/] {exc}")
raise typer.Exit(2) from exc
+
+def _report_config_support(cfg) -> None:
+ """#755 — report the settings this config sets that its backend never reads.
+
+ Only fields the user wrote and switched on are listed (a ``false`` or unset
+ value is not; ``seed: 0`` is). A wall of 275 rows is not a pre-flight check,
+ and the fields sitting at their schema default or written in their off
+ position are not what anyone came here to ask about.
+ """
+ from soup_cli.config.backend_support import (
+ DEFAULT_BACKEND,
+ DEFAULT_MODALITY,
+ check_config,
+ unsupported_for,
+ )
+
backend = getattr(cfg, "backend", DEFAULT_BACKEND)
modality = getattr(cfg, "modality", DEFAULT_MODALITY)
gaps = check_config(cfg)
diff --git a/tests/test_doctor.py b/tests/test_doctor.py
index 61bc155d..dc1a9478 100644
--- a/tests/test_doctor.py
+++ b/tests/test_doctor.py
@@ -1026,3 +1026,149 @@ def test_doctor_nccl_mocked_success():
assert result.exit_code == 0
assert "Measuring NCCL bandwidth" in _strip_ansi(result.output)
assert "Result (H100 over NVLINK)" in _strip_ansi(result.output)
+
+
+# --- Doctor --config tests (#2100) ---
+
+
+def _mock_clean_doctor_env(monkeypatch, missing: tuple[str, ...] | list[str] = ()):
+ installed = {
+ "pydantic": "2.10.0",
+ "typer": "0.15.4",
+ "rich": "13.0.0",
+ "pyyaml": "6.0",
+ "plotext": "5.2.0",
+ "torch": "2.6.0",
+ "transformers": "5.16.1",
+ "peft": "0.20.0",
+ "trl": "0.29.0",
+ "datasets": "2.14.0",
+ "bitsandbytes": "0.41.0",
+ "accelerate": "0.27.0",
+ }
+ for pkg in missing:
+ installed.pop(pkg, None)
+ monkeypatch.setattr(
+ "soup_cli.commands.doctor._installed_version_str",
+ lambda imp, pkg: installed.get(pkg),
+ )
+ monkeypatch.setattr(
+ "soup_cli.commands.doctor._check_torchvision_compat",
+ lambda issues: None,
+ )
+
+
+def test_doctor_refused_config_does_not_print_all_checks_passed(tmp_path, monkeypatch):
+ """A refused config must not print 'All checks passed!' before exit 2 (#2100)."""
+ _mock_clean_doctor_env(monkeypatch)
+ bad_config = tmp_path / "soup.yaml"
+ bad_config.write_text(
+ "base: meta-llama/Llama-3.1-8B-Instruct\nunknown_key: true\n",
+ encoding="utf-8",
+ )
+ result = runner.invoke(app, ["doctor", "--config", str(bad_config)])
+ assert result.exit_code == 2, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "All checks passed!" not in out
+ assert "unknown config key 'unknown_key'" in out
+ assert "Config could not be loaded" in out
+
+
+def test_doctor_missing_package_and_refused_config_shows_issues_and_refusal(tmp_path, monkeypatch):
+ """Environment issues are not hidden when a refused config exits 2 (#2100)."""
+ _mock_clean_doctor_env(monkeypatch, missing=["peft"])
+ bad_config = tmp_path / "soup.yaml"
+ bad_config.write_text(
+ "base: meta-llama/Llama-3.1-8B-Instruct\nunknown_key: true\n",
+ encoding="utf-8",
+ )
+ result = runner.invoke(app, ["doctor", "--config", str(bad_config)])
+ assert result.exit_code == 2, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "Found 1 issue(s):" in out
+ assert 'pip install "soup-cli[train]"' in out
+ assert "unknown config key 'unknown_key'" in out
+ assert "Config could not be loaded" in out
+ assert "All checks passed!" not in out
+
+
+def test_doctor_missing_package_and_valid_config_omits_summary_and_reports_config(
+ sample_config, monkeypatch
+):
+ """When an issue exists, a valid config is reported but summary is omitted (#2100)."""
+ _mock_clean_doctor_env(monkeypatch, missing=["peft"])
+ result = runner.invoke(app, ["doctor", "--config", str(sample_config)])
+ assert result.exit_code == 0, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "Found 1 issue(s):" in out
+ assert "All checks passed!" not in out
+ assert "Config check" in out
+ assert out.index("Found 1 issue(s):") < out.index("Config check")
+
+
+def test_doctor_valid_config_prints_all_checks_passed(sample_config, monkeypatch):
+ """A valid config check allows 'All checks passed!' on a healthy environment (#2100)."""
+ _mock_clean_doctor_env(monkeypatch)
+ result = runner.invoke(app, ["doctor", "--config", str(sample_config)])
+ assert result.exit_code == 0, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "All checks passed!" in out
+ assert "Config check" in out
+
+
+def test_doctor_loadable_config_with_gaps_prints_findings_after_summary(tmp_path, monkeypatch):
+ """Config check findings for a loadable config are printed after summary (#2100)."""
+ _mock_clean_doctor_env(monkeypatch)
+ gap_config = tmp_path / "soup.yaml"
+ gap_config.write_text(
+ "base: Qwen/Qwen2-VL-7B-Instruct\n"
+ "task: sft\n"
+ "modality: vision\n"
+ "data:\n"
+ " train: ./data.jsonl\n"
+ " format: llava\n"
+ " mask_history: true\n",
+ encoding="utf-8",
+ )
+ result = runner.invoke(app, ["doctor", "--config", str(gap_config)])
+ assert result.exit_code == 0, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "All checks passed!" in out
+ assert "Config check" in out
+ assert out.index("All checks passed!") < out.index("Config check")
+
+
+def test_doctor_without_config_unchanged(monkeypatch):
+ """Without --config, doctor still prints 'All checks passed!' and exits 0 (#2100)."""
+ _mock_clean_doctor_env(monkeypatch)
+ result = runner.invoke(app, ["doctor"])
+ assert result.exit_code == 0, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "All checks passed!" in out
+
+
+def test_doctor_nonexistent_config_exits_2(tmp_path, monkeypatch):
+ """A nonexistent config path prints 'Config not found' and exits 2 (#2100)."""
+ _mock_clean_doctor_env(monkeypatch)
+ missing_config = tmp_path / "does_not_exist.yaml"
+ result = runner.invoke(app, ["doctor", "--config", str(missing_config)])
+ assert result.exit_code == 2, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "Config not found:" in out
+ assert "All checks passed!" not in out
+
+
+def test_doctor_load_error_exception_exits_2(monkeypatch):
+ """An unexpected exception during config load prints error detail and exits 2 (#2100)."""
+ _mock_clean_doctor_env(monkeypatch)
+
+ def _fail_load(path):
+ raise RuntimeError("disk read failed")
+
+ monkeypatch.setattr("soup_cli.config.loader.load_config", _fail_load)
+ result = runner.invoke(app, ["doctor", "--config", "unreadable.yaml"])
+ assert result.exit_code == 2, (result.output, repr(result.exception))
+ out = _strip_ansi(result.output)
+ assert "Config could not be loaded: disk read failed" in out
+ assert "All checks passed!" not in out
+