Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
fix(doctor): validate config before printing environment success summary (#2100) (#2106)
Nariman Abdikarim7a6edd9b61a0
diff --git a/changelog.d/0.76.0/2106.fixed.md b/changelog.d/0.76.0/2106.fixed.mdnew file mode 100644index 00000000..a95b6178--- /dev/null+++ b/changelog.d/0.76.0/2106.fixed.md@@ -0,0 +1 @@+- Run config check before environment summary in `soup doctor` so a refused or invalid config does not falsely print "All checks passed!" before terminating with exit 2 (#2100, #2106 by @Nari-Ab).diff --git a/src/soup_cli/commands/doctor.py b/src/soup_cli/commands/doctor.pyindex 836b4c7b..9fa66063 100644--- a/src/soup_cli/commands/doctor.py+++ b/src/soup_cli/commands/doctor.py@@ -327,14 +327,22 @@ def doctor(console.print(f"[dim] pip install {escape(' '.join(fix_parts))}[/]", highlight=False)- else:++ loaded_cfg = None+ if config:+ # Load and validate the config before the environment summary (#2100).+ # If the config is refused or cannot be loaded, doctor terminates with exit 2+ # before asserting that "All checks passed!".+ loaded_cfg = _load_doctor_config(config)++ if not issues:console.print("\n[bold green]All checks passed![/] Your environment is ready.")# After the environment summary on purpose: "All checks passed" reports on# the environment, and printing config findings above it read as though the# green line covered them too.- if config:- _check_config_support(config)+ if loaded_cfg is not None:+ _report_config_support(loaded_cfg)console.print(f"\n[dim]GitHub: [link={GITHUB_URL}]{GITHUB_URL}[/link][/]")@@ -360,25 +368,12 @@ def _installed_version_str(import_name: str, pkg_name: str) -> str | None:return str(version)-def _check_config_support(config_path: str) -> None:- """#755 — report the settings this config sets that its backend never reads.-- Only fields the user wrote and switched on are listed (a ``false`` or unset- value is not; ``seed: 0`` is). A wall of 275 rows is not a pre-flight check,- and the fields sitting at their schema default or written in their off- position are not what anyone came here to ask about.- """- from soup_cli.config.backend_support import (- DEFAULT_BACKEND,- DEFAULT_MODALITY,- check_config,- unsupported_for,- )-+def _load_doctor_config(config_path: str):+ """Load config for doctor, exiting with code 2 on refusal or load error (#2100)."""try:from soup_cli.config.loader import load_config- cfg = load_config(config_path)+ return load_config(config_path)except FileNotFoundError as exc:console.print(f"\n[red]Config not found:[/] {config_path}")# Non-zero deliberately: this leg is meant to be gate-able in CI, and a@@ -397,6 +392,22 @@ def _check_config_support(config_path: str) -> None:console.print(f"\n[red]Config could not be loaded:[/] {exc}")raise typer.Exit(2) from exc++def _report_config_support(cfg) -> None:+ """#755 — report the settings this config sets that its backend never reads.++ Only fields the user wrote and switched on are listed (a ``false`` or unset+ value is not; ``seed: 0`` is). A wall of 275 rows is not a pre-flight check,+ and the fields sitting at their schema default or written in their off+ position are not what anyone came here to ask about.+ """+ from soup_cli.config.backend_support import (+ DEFAULT_BACKEND,+ DEFAULT_MODALITY,+ check_config,+ unsupported_for,+ )+backend = getattr(cfg, "backend", DEFAULT_BACKEND)modality = getattr(cfg, "modality", DEFAULT_MODALITY)gaps = check_config(cfg)diff --git a/tests/test_doctor.py b/tests/test_doctor.pyindex 61bc155d..dc1a9478 100644--- a/tests/test_doctor.py+++ b/tests/test_doctor.py@@ -1026,3 +1026,149 @@ def test_doctor_nccl_mocked_success():assert result.exit_code == 0assert "Measuring NCCL bandwidth" in _strip_ansi(result.output)assert "Result (H100 over NVLINK)" in _strip_ansi(result.output)+++# --- Doctor --config tests (#2100) ---+++def _mock_clean_doctor_env(monkeypatch, missing: tuple[str, ...] | list[str] = ()):+ installed = {+ "pydantic": "2.10.0",+ "typer": "0.15.4",+ "rich": "13.0.0",+ "pyyaml": "6.0",+ "plotext": "5.2.0",+ "torch": "2.6.0",+ "transformers": "5.16.1",+ "peft": "0.20.0",+ "trl": "0.29.0",+ "datasets": "2.14.0",+ "bitsandbytes": "0.41.0",+ "accelerate": "0.27.0",+ }+ for pkg in missing:+ installed.pop(pkg, None)+ monkeypatch.setattr(+ "soup_cli.commands.doctor._installed_version_str",+ lambda imp, pkg: installed.get(pkg),+ )+ monkeypatch.setattr(+ "soup_cli.commands.doctor._check_torchvision_compat",+ lambda issues: None,+ )+++def test_doctor_refused_config_does_not_print_all_checks_passed(tmp_path, monkeypatch):+ """A refused config must not print 'All checks passed!' before exit 2 (#2100)."""+ _mock_clean_doctor_env(monkeypatch)+ bad_config = tmp_path / "soup.yaml"+ bad_config.write_text(+ "base: meta-llama/Llama-3.1-8B-Instruct\nunknown_key: true\n",+ encoding="utf-8",+ )+ result = runner.invoke(app, ["doctor", "--config", str(bad_config)])+ assert result.exit_code == 2, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "All checks passed!" not in out+ assert "unknown config key 'unknown_key'" in out+ assert "Config could not be loaded" in out+++def test_doctor_missing_package_and_refused_config_shows_issues_and_refusal(tmp_path, monkeypatch):+ """Environment issues are not hidden when a refused config exits 2 (#2100)."""+ _mock_clean_doctor_env(monkeypatch, missing=["peft"])+ bad_config = tmp_path / "soup.yaml"+ bad_config.write_text(+ "base: meta-llama/Llama-3.1-8B-Instruct\nunknown_key: true\n",+ encoding="utf-8",+ )+ result = runner.invoke(app, ["doctor", "--config", str(bad_config)])+ assert result.exit_code == 2, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "Found 1 issue(s):" in out+ assert 'pip install "soup-cli[train]"' in out+ assert "unknown config key 'unknown_key'" in out+ assert "Config could not be loaded" in out+ assert "All checks passed!" not in out+++def test_doctor_missing_package_and_valid_config_omits_summary_and_reports_config(+ sample_config, monkeypatch+):+ """When an issue exists, a valid config is reported but summary is omitted (#2100)."""+ _mock_clean_doctor_env(monkeypatch, missing=["peft"])+ result = runner.invoke(app, ["doctor", "--config", str(sample_config)])+ assert result.exit_code == 0, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "Found 1 issue(s):" in out+ assert "All checks passed!" not in out+ assert "Config check" in out+ assert out.index("Found 1 issue(s):") < out.index("Config check")+++def test_doctor_valid_config_prints_all_checks_passed(sample_config, monkeypatch):+ """A valid config check allows 'All checks passed!' on a healthy environment (#2100)."""+ _mock_clean_doctor_env(monkeypatch)+ result = runner.invoke(app, ["doctor", "--config", str(sample_config)])+ assert result.exit_code == 0, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "All checks passed!" in out+ assert "Config check" in out+++def test_doctor_loadable_config_with_gaps_prints_findings_after_summary(tmp_path, monkeypatch):+ """Config check findings for a loadable config are printed after summary (#2100)."""+ _mock_clean_doctor_env(monkeypatch)+ gap_config = tmp_path / "soup.yaml"+ gap_config.write_text(+ "base: Qwen/Qwen2-VL-7B-Instruct\n"+ "task: sft\n"+ "modality: vision\n"+ "data:\n"+ " train: ./data.jsonl\n"+ " format: llava\n"+ " mask_history: true\n",+ encoding="utf-8",+ )+ result = runner.invoke(app, ["doctor", "--config", str(gap_config)])+ assert result.exit_code == 0, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "All checks passed!" in out+ assert "Config check" in out+ assert out.index("All checks passed!") < out.index("Config check")+++def test_doctor_without_config_unchanged(monkeypatch):+ """Without --config, doctor still prints 'All checks passed!' and exits 0 (#2100)."""+ _mock_clean_doctor_env(monkeypatch)+ result = runner.invoke(app, ["doctor"])+ assert result.exit_code == 0, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "All checks passed!" in out+++def test_doctor_nonexistent_config_exits_2(tmp_path, monkeypatch):+ """A nonexistent config path prints 'Config not found' and exits 2 (#2100)."""+ _mock_clean_doctor_env(monkeypatch)+ missing_config = tmp_path / "does_not_exist.yaml"+ result = runner.invoke(app, ["doctor", "--config", str(missing_config)])+ assert result.exit_code == 2, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "Config not found:" in out+ assert "All checks passed!" not in out+++def test_doctor_load_error_exception_exits_2(monkeypatch):+ """An unexpected exception during config load prints error detail and exits 2 (#2100)."""+ _mock_clean_doctor_env(monkeypatch)++ def _fail_load(path):+ raise RuntimeError("disk read failed")++ monkeypatch.setattr("soup_cli.config.loader.load_config", _fail_load)+ result = runner.invoke(app, ["doctor", "--config", "unreadable.yaml"])+ assert result.exit_code == 2, (result.output, repr(result.exception))+ out = _strip_ansi(result.output)+ assert "Config could not be loaded: disk read failed" in out+ assert "All checks passed!" not in out+