The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
test: cover offline model selector operator workflows
haelyra0c35647012e7
diff --git a/tests/test_selector_workflows.py b/tests/test_selector_workflows.pynew file mode 100644index 000000000..aa55f9382--- /dev/null+++ b/tests/test_selector_workflows.py@@ -0,0 +1,304 @@+"""Offline operator workflows for the public selector API and CLI entries."""++import io+import json+import os+import socket+import subprocess+import sys+from collections.abc import Iterator+from dataclasses import dataclass+from pathlib import Path++import pytest++from llm.cli.selector import interactive_select, main+from llm.providers import get_provider++SOURCE = Path(__file__).resolve().parents[1] / "src"+HOME_CONFIG = "LLM_PROVIDER=ollama\nLLM_MODEL=home-must-stay-untouched\n"+NETWORK_GUARD = """+import sys+def deny_network(event, args):+ if event in {"socket.connect", "socket.getaddrinfo", "socket.sendto"}:+ raise RuntimeError("Selector workflows must remain offline")+sys.addaudithook(deny_network)+"""+++@dataclass(frozen=True)+class Operator:+ project: Path+ env: dict[str, str]++ def run(+ self, args: list[str], answers: str = ""+ ) -> subprocess.CompletedProcess[str]:+ return subprocess.run(+ [sys.executable, *args],+ input=answers,+ cwd=self.project,+ env=self.env,+ text=True,+ capture_output=True,+ timeout=15,+ check=True,+ )++ def selection(self) -> list[str]:+ """Resolve saved configuration in a fresh process without inference."""+ result = self.run(+ [+ "-c",+ "import json; from llm import get_provider; p = get_provider(); "+ "print(json.dumps([p.provider_type.value, p.get_default_model()]))",+ ]+ )+ selection: list[str] = json.loads(result.stdout)+ return selection+++def operator(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[Operator]:+ home = tmp_path / "home"+ project = tmp_path / "project"+ guard = tmp_path / "offline"+ for directory in (home, project, guard):+ directory.mkdir()+ home.joinpath(".llm.env").write_text(HOME_CONFIG)+ guard.joinpath("sitecustomize.py").write_text(NETWORK_GUARD)+ env = {+ "HOME": str(home),+ "PATH": os.defpath,+ "PYTHONPATH": os.pathsep.join((str(guard), str(SOURCE))),+ "PYTHONUTF8": "1",+ "ANTHROPIC_API_KEY": "offline-test-key",+ "OPENAI_API_KEY": "offline-test-key",+ }+ monkeypatch.chdir(project)+ for name in tuple(os.environ):+ if name.startswith(("LLM_", "OLLAMA_", "ATLAS_", "ASTRAFLOW_")):+ monkeypatch.delenv(name)+ # Record absent values too: persist=True writes directly to os.environ.+ for name in ("LLM_PROVIDER", "LLM_MODEL"):+ monkeypatch.setenv(name, "")+ monkeypatch.delenv(name)+ for name in ("HOME", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"):+ monkeypatch.setenv(name, env[name])++ def deny_network(*args: object, **kwargs: object) -> None:+ pytest.fail("Selector workflow attempted a network connection")++ monkeypatch.setattr(socket.socket, "connect", deny_network)+ monkeypatch.setattr(socket.socket, "connect_ex", deny_network)+ monkeypatch.setattr(socket, "getaddrinfo", deny_network)+ yield Operator(project, env)+ assert home.joinpath(".llm.env").read_text() == HOME_CONFIG++[email protected]("entry", ["module", "console"])+def test_real_cli_creates_loads_and_replaces_project_selection(+ operator: Operator, entry: str+) -> None:+ if entry == "console":+ script = Path(sys.executable).parent / "llm-select"+ if not script.is_file():+ pytest.skip(+ "Installed llm-select console entry unavailable; module entry still exercised"+ )+ args = [str(script)]+ else:+ args = ["-m", "llm"]++ assert operator.selection() == ["claude", "claude-sonnet-4-6"]+ origin = operator.run(["-c", "import llm; print(llm.__file__)"])+ assert Path(origin.stdout.strip()).resolve() == SOURCE / "llm" / "__init__.py"+ first = operator.run(args, "3\n2\n")+ assert "Available Providers:" in first.stdout+ assert "Local Ollama models" in first.stdout+ assert "Available Models:" in first.stdout+ assert "Mistral - Fast & efficient" in first.stdout+ assert "Ready to use!" in first.stdout+ assert "export LLM_PROVIDER=ollama" in first.stdout+ assert "export LLM_MODEL=mistral" in first.stdout+ assert first.stderr == ""+ config = operator.project / ".llm.env"+ assert config.read_text() == "LLM_PROVIDER=ollama\nLLM_MODEL=mistral\n"+ assert operator.selection() == ["ollama", "mistral"]++ second = operator.run(args, "2\n2\n")+ assert "GPT-4o-mini - Fast & affordable" in second.stdout+ assert second.stderr == ""+ assert config.read_text() == "LLM_PROVIDER=openai\nLLM_MODEL=gpt-4o-mini\n"+ assert operator.selection() == ["openai", "gpt-4o-mini"]+ assert (+ "LLM_PROVIDER" not in os.environ+ ) # A child cannot select for its parent shell.+++ "provider_number,provider,model,description",+ [+ (1, "claude", "claude-sonnet-4-6", "Claude Sonnet 4.6 - Balanced"),+ (2, "openai", "gpt-4o-mini", "GPT-4o-mini - Fast & affordable"),+ (3, "ollama", "mistral", "Mistral - Fast & efficient"),+ ],+)+def test_default_catalog_selection_is_saved_without_changing_current_environment(+ operator: Operator,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ provider_number: int,+ provider: str,+ model: str,+ description: str,+) -> None:+ monkeypatch.setattr(sys, "stdin", io.StringIO(f" {provider_number} \n 2 \n"))++ assert interactive_select() == (provider, model)++ output = capsys.readouterr().out+ assert "LLM Provider Selector" in output+ assert description in output+ assert "Config saved to" in output+ assert "Config loaded to current session" not in output+ assert "LLM_MODEL" not in os.environ+ assert operator.selection() == [provider, model]++[email protected]("persist", [False, True])+def test_custom_catalog_updates_saved_selection_and_optionally_current_process(+ operator: Operator,+ monkeypatch: pytest.MonkeyPatch,+ persist: bool,+) -> None:+ monkeypatch.setenv("LLM_PROVIDER", "ollama")+ monkeypatch.setenv("LLM_MODEL", "codellama")+ operator.project.joinpath(".llm.env").write_text(+ "LLM_PROVIDER=ollama\nLLM_MODEL=mistral\n"+ )+ monkeypatch.setattr(sys, "stdin", io.StringIO("1\n1\n"))++ assert interactive_select(+ providers=[("ollama", "Operator local catalog")],+ models_per_provider={+ "ollama": [("Custom/MixedCase-model", "Installed local model")]+ },+ persist=persist,+ ) == ("ollama", "Custom/MixedCase-model")++ assert operator.selection() == ["ollama", "Custom/MixedCase-model"]+ assert get_provider().get_default_model() == (+ "Custom/MixedCase-model" if persist else "codellama"+ )+++def test_cli_reprompts_for_malformed_and_out_of_range_choices(+ operator: Operator,+) -> None:+ result = operator.run(["-m", "llm"], "text\n0\n-1\n99\n 3 \n1.5\n0\n99\n 2 \n")++ assert result.stdout.count("Please enter a number.") == 2+ assert result.stdout.count("Invalid selection. Try again.") == 5+ assert "Ready to use!" in result.stdout+ assert result.stderr == ""+ assert operator.selection() == ["ollama", "mistral"]++[email protected]("answers", [" \n", "3\n\t\n"], ids=["provider", "model"])+ "existing", [False, True], ids=["new-project", "saved-project"]+)+def test_cli_cancellation_preserves_existing_config_or_creates_nothing(+ operator: Operator,+ answers: str,+ existing: bool,+) -> None:+ config = operator.project / ".llm.env"+ original = b"# operator notes\nLLM_PROVIDER=ollama\nLLM_MODEL=codellama\n"+ if existing:+ config.write_bytes(original)++ result = operator.run(["-m", "llm"], answers)++ assert "Selection cancelled." in result.stdout+ assert "Config saved" not in result.stdout+ assert "Ready to use!" not in result.stdout+ assert result.stderr == ""+ if existing:+ assert config.read_bytes() == original+ assert operator.selection() == ["ollama", "codellama"]+ else:+ assert not config.exists()+++ "providers,models,answers,message",+ [+ ([], {}, "", "No providers available."),+ ([("ollama", "Local")], {}, "1\n", "No models available."),+ ([("custom", "Operator catalog")], None, "1\n", "No models available."),+ (None, None, "\n", "Available Providers:"),+ (None, None, "3\n\n", "Available Models:"),+ ],+)+def test_interactive_cancellation_handles_empty_catalogs_without_overwriting_config(+ operator: Operator,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ providers: list[tuple[str, str]] | None,+ models: dict[str, list[tuple[str, str]]] | None,+ answers: str,+ message: str,+) -> None:+ config = operator.project / ".llm.env"+ original = "LLM_PROVIDER=ollama\nLLM_MODEL=codellama\n"+ config.write_text(original)+ monkeypatch.setattr(sys, "stdin", io.StringIO(answers))++ assert interactive_select(providers, models, persist=True) is None++ assert message in capsys.readouterr().out+ assert config.read_text() == original+ assert "LLM_PROVIDER" not in os.environ+++ "answers", ["bad\n0\n3\nwrong\n99\n2\n", "\n"], ids=["select", "cancel"]+)+def test_public_main_sets_current_selection_or_exits_cleanly(+ operator: Operator,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ answers: str,+) -> None:+ monkeypatch.setattr(sys, "stdin", io.StringIO(answers))+ if answers == "\n":+ with pytest.raises(SystemExit) as cancelled:+ main()+ assert cancelled.value.code == 0+ assert "Selection cancelled." in capsys.readouterr().out+ assert not operator.project.joinpath(".llm.env").exists()+ assert "LLM_PROVIDER" not in os.environ+ else:+ main()+ output = capsys.readouterr().out+ assert "Ready to use!" in output+ assert "Config loaded to current session" in output+ assert os.environ["LLM_PROVIDER"] == "ollama"+ assert os.environ["LLM_MODEL"] == "mistral"+ assert get_provider().get_default_model() == "mistral"+++def test_subprocess_network_guard_is_active(operator: Operator) -> None:+ """Prove the CLI child environment would reject accidental provider access."""+ result = operator.run(+ [+ "-c",+ "import socket; "+ "\ntry: socket.create_connection(('127.0.0.1', 9))"+ "\nexcept RuntimeError as error: print(error)"+ "\nelse: raise AssertionError('Network guard missing')",+ ]+ )+ assert "Selector workflows must remain offline" in result.stdout