Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

16.2 KB
Raw Blame
from __future__ import annotations

import os
import shutil
from pathlib import Path
from typing import TYPE_CHECKING, Any

if TYPE_CHECKING:
    from pytest_mock import MockerFixture

    from pipelex.hosted.run_config import RunExecution

from pipelex.cli.commands.init.setup_path import SetupPath, read_run_execution
from pipelex.cli.commands.login.command import LoginOutcome
from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
from pipelex.kit.paths import get_kit_configs_dir
from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit

#: What a person types at "Where should your runs execute?" to pick each path.
SETUP_PATH_ANSWERS: dict[SetupPath, str] = {SetupPath.HOSTED: "1", SetupPath.LOCAL: "2"}


def get_backend_indices_helper(backends_toml_path: str, backend_names: list[str]) -> list[int]:
    """Get 1-based indices for given backend names from backends.toml.

    Args:
        backends_toml_path: Path to backends.toml file.
        backend_names: List of backend keys to find indices for.

    Returns:
        List of 1-based indices corresponding to the backend names.
    """
    toml_doc = load_toml_with_tomlkit(backends_toml_path)
    backend_list: list[str] = [key for key in toml_doc if key != "internal"]
    indices: list[int] = []
    for name in backend_names:
        if name in backend_list:
            # 1-based index for user input
            indices.append(backend_list.index(name) + 1)
    return indices


def verify_backends_toml(backends_toml_path: str, expected_enabled: list[str]) -> None:
    """Verify backends.toml has expected backends enabled.

    Args:
        backends_toml_path: Path to backends.toml file.
        expected_enabled: List of backend keys expected to be enabled.
    """
    toml_doc = load_toml_with_tomlkit(backends_toml_path)

    for backend_key in toml_doc:
        if backend_key == "internal":
            # Internal should always be enabled
            assert toml_doc[backend_key]["enabled"] is True  # type: ignore[index]
        elif backend_key in expected_enabled:
            assert toml_doc[backend_key]["enabled"] is True, f"Expected {backend_key} to be enabled"  # type: ignore[index]
        # Check if enabled field exists before asserting
        elif "enabled" in toml_doc[backend_key]:  # type: ignore[operator]
            assert toml_doc[backend_key]["enabled"] is False, f"Expected {backend_key} to be disabled"  # type: ignore[index]


def verify_routing_profile(
    routing_profiles_toml_path: str,
    expected_active: str,
    expected_default: str | None = None,
    expected_fallback_order: list[str] | None = None,
) -> None:
    """Verify routing_profiles.toml has expected configuration.

    Args:
        routing_profiles_toml_path: Path to routing_profiles.toml file.
        expected_active: Expected active profile name.
        expected_default: Expected default backend (optional).
        expected_fallback_order: Expected fallback order (optional).
    """
    toml_doc = load_toml_with_tomlkit(routing_profiles_toml_path)

    assert toml_doc["active"] == expected_active, f"Expected active profile '{expected_active}', got '{toml_doc['active']}'"

    if expected_default is not None or expected_fallback_order is not None:
        profiles: dict[str, Any] = toml_doc.get("profiles", {})  # type: ignore[assignment]
        assert expected_active in profiles, f"Profile '{expected_active}' not found"

        profile = profiles[expected_active]  # type: ignore[index]

        if expected_default is not None:
            assert profile.get("default") == expected_default, f"Expected default '{expected_default}'"  # type: ignore[union-attr]

        if expected_fallback_order is not None:
            actual_fallback = profile.get("fallback_order")  # type: ignore[union-attr]
            assert actual_fallback == expected_fallback_order, f"Expected fallback_order {expected_fallback_order}, got {actual_fallback}"


def verify_telemetry_config(telemetry_config_path: str, expected_mode: str) -> None:
    """Verify telemetry.toml has expected mode.

    Args:
        telemetry_config_path: Path to telemetry.toml file.
        expected_mode: Expected telemetry mode ("off", "anonymous", or "identified").
    """
    toml_doc = load_toml_with_tomlkit(telemetry_config_path)
    actual_mode = toml_doc["custom_posthog"]["mode"]  # type: ignore[index]
    assert actual_mode == expected_mode, f"Expected posthog.mode '{expected_mode}', got '{actual_mode}'"


class MockedInitEnvironment:
    """Encapsulates all mocking setup for init command testing.

    This class manages:
    - File system setup (tmp_path, .pipelex directory)
    - Mocking of Console, Prompt, Confirm
    - User input simulation
    - Configuration verification
    """

    def __init__(self, tmp_path: Path, mocker: MockerFixture):
        """Initialize mocked environment.

        Args:
            tmp_path: Pytest temporary path fixture.
            mocker: Pytest mocker fixture.
        """
        self.tmp_path = tmp_path
        self.mocker = mocker
        self.pipelex_dir = tmp_path / ".pipelex"
        self.inference_dir = self.pipelex_dir / "inference"

        # Store mocks for later use
        self.mock_console: Any = None
        self.mock_prompt_ask: Any = None
        self.mock_confirm_ask: Any = None
        self.mock_config_manager: Any = None
        self.mock_login: Any = None
        self.mock_browser_open: Any = None

        # Input sequences
        self.prompt_inputs: list[str] = []
        self.confirm_inputs: list[bool] = []

    def setup_empty_dir(self) -> None:
        """Set up an empty .pipelex directory structure."""
        self.pipelex_dir.mkdir(exist_ok=True)
        self.inference_dir.mkdir(exist_ok=True)

    def setup_with_configs(self, include_backends: bool = True, include_routing: bool = True, include_telemetry: bool = False) -> None:
        """Set up .pipelex directory with config files from kit.

        Args:
            include_backends: Whether to include backends.toml and backends/ subdirectory.
            include_routing: Whether to include routing_profiles.toml.
            include_telemetry: Whether to include telemetry.toml.

        Raises:
            FileNotFoundError: If any required kit config files are missing (raised by shutil.copy2).
        """
        # Create directory structure
        self.pipelex_dir.mkdir(exist_ok=True)
        self.inference_dir.mkdir(exist_ok=True)

        kit_configs_dir = Path(str(get_kit_configs_dir()))

        # Always copy main pipelex.toml (needed for init_config checks)
        shutil.copy2(kit_configs_dir / "pipelex.toml", self.pipelex_dir / "pipelex.toml")

        # Always copy deck/ subdirectory (needed for init_config checks)
        kit_deck_dir = kit_configs_dir / "inference" / "deck"
        target_deck_dir = self.inference_dir / "deck"
        target_deck_dir.mkdir(exist_ok=True)
        for deck_file in kit_deck_dir.glob("*.toml"):
            shutil.copy2(deck_file, target_deck_dir / deck_file.name)

        if include_backends:
            # Copy backends.toml
            shutil.copy2(kit_configs_dir / "inference" / "backends.toml", self.inference_dir / "backends.toml")

            # Copy backends/ subdirectory (needed for init_config checks)
            kit_backends_dir = kit_configs_dir / "inference" / "backends"
            target_backends_dir = self.inference_dir / "backends"
            target_backends_dir.mkdir(exist_ok=True)
            for backend_file in kit_backends_dir.glob("*.toml"):
                shutil.copy2(backend_file, target_backends_dir / backend_file.name)

        if include_routing:
            # Copy routing_profiles.toml
            shutil.copy2(kit_configs_dir / "inference" / "routing_profiles.toml", self.inference_dir / "routing_profiles.toml")

        if include_telemetry:
            shutil.copy2(kit_configs_dir / "telemetry.toml", self.pipelex_dir / "telemetry.toml")

    def mock_config_manager_paths(self) -> None:
        """Mock config_manager to use tmp_path."""
        self.mock_config_manager = self.mocker.MagicMock()
        self.mock_config_manager.pipelex_config_dir = self.pipelex_dir
        self.mock_config_manager.global_config_dir = self.pipelex_dir
        self.mock_config_manager.project_config_dir = self.pipelex_dir
        self.mock_config_manager.project_root = self.tmp_path
        self.mock_config_manager.backends_file_path = self.inference_dir / "backends.toml"
        self.mock_config_manager.backends_dir_path = self.inference_dir / "backends"
        self.mock_config_manager.routing_profiles_file_path = self.inference_dir / "routing_profiles.toml"
        self.mock_config_manager.model_decks_dir_path = self.inference_dir / "deck"

        # Patch all locations where config_manager is used
        self.mocker.patch("pipelex.cli.commands.init.command.config_manager", self.mock_config_manager)
        self.mocker.patch("pipelex.cli.commands.init.backends.config_manager", self.mock_config_manager)
        self.mocker.patch("pipelex.cli.commands.init.routing.config_manager", self.mock_config_manager)
        self.mocker.patch("pipelex.cli.commands.init.config_files.config_manager", self.mock_config_manager)
        # The credentials file, where `pipelex login` saves its key, lives in the home directory this test stands in
        self.mocker.patch("pipelex.cli.commands.init.credentials.config_manager", self.mock_config_manager)

    def mock_console_outputs(self) -> None:
        """Mock Console handling to suppress output."""
        self.mock_console = self.mocker.MagicMock()

        # Patch modules that use the Pipelex hub console provider
        self.mocker.patch("pipelex.runtime_hub.get_console", return_value=self.mock_console)
        self.mocker.patch("pipelex.cli.commands.init.command.get_console", return_value=self.mock_console)
        self.mocker.patch("pipelex.cli.commands.init.backends.get_console", return_value=self.mock_console)
        self.mocker.patch("pipelex.cli.commands.init.routing.get_console", return_value=self.mock_console)

        # Patch UI modules that still instantiate Console directly
        self.mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Console")
        self.mocker.patch("pipelex.cli.commands.init.ui.routing_ui.Console")

    def add_prompt_input(self, value: str) -> None:
        """Add a prompt input to the sequence.

        Args:
            value: The input value to simulate.
        """
        self.prompt_inputs.append(value)

    def choose_setup_path(self, setup_path: SetupPath) -> None:
        """Answer "Where should your runs execute?" with this path.

        Args:
            setup_path: The path to pick; Enter would take the hosted Pipelex API.
        """
        self.prompt_inputs.append(SETUP_PATH_ANSWERS[setup_path])

    def choose_local(self) -> None:
        """Answer "Where should your runs execute?" with this machine, the path every backend test walks."""
        self.choose_setup_path(SetupPath.LOCAL)

    def add_confirm_input(self, value: bool) -> None:
        """Add a confirm input to the sequence.

        Args:
            value: The boolean confirmation to simulate.
        """
        self.confirm_inputs.append(value)

    def setup_mocks(self) -> None:
        """Set up all mocks with configured inputs."""
        self.mock_config_manager_paths()
        self.mock_console_outputs()

        def prompt_side_effect(*args: Any, **_kwargs: Any) -> str:
            if not self.prompt_inputs:
                question = str(args[0]) if args else "<unknown prompt>"
                msg = f"Unexpected prompt without predefined input: {question}"
                raise AssertionError(msg)
            return self.prompt_inputs.pop(0)

        def confirm_side_effect(*args: Any, **_kwargs: Any) -> bool:
            if not self.confirm_inputs:
                question = str(args[0]) if args else "<unknown confirmation>"
                msg = f"Unexpected confirm without predefined input: {question}"
                raise AssertionError(msg)
            return self.confirm_inputs.pop(0)

        self.mock_prompt_ask = self.mocker.patch("rich.prompt.Prompt.ask", side_effect=prompt_side_effect)
        self.mock_confirm_ask = self.mocker.patch("rich.prompt.Confirm.ask", side_effect=confirm_side_effect)

        # Mock typer.echo to suppress output
        self.mocker.patch("typer.echo")

        # Mock prompt_credentials to avoid env-dependent Prompt.ask calls for missing API keys
        self.mocker.patch("pipelex.cli.commands.init.command.prompt_credentials")

        # The IDE extension suggestion asks to install into the editor it finds on this machine, so it would
        # consume a queued answer and install into the real editor wherever the extension is missing
        self.mocker.patch("pipelex.cli.commands.init.backends.suggest_extension_install_if_needed")
        self.mocker.patch("pipelex.cli.commands.init.command.suggest_extension_install_if_needed")

        # The hosted path signs in through the browser: no test opens one or reaches the hosted API. The login
        # reports a saved key by default; a test sets `mock_login.return_value` for another outcome.
        self.mock_login = self.mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser", return_value=LoginOutcome.SAVED)
        self.mock_browser_open = self.mocker.patch("pipelex.cli.commands.login.command.webbrowser.open")

        # Start with no Pipelex API key, whatever the developer's shell exports, and restore the environment
        # afterwards, since a login sets the key in it directly
        self.mocker.patch.dict(os.environ)
        os.environ.pop(PIPELEX_API_KEY_ENV_KEY, None)

    def get_backend_indices(self, backend_names: list[str]) -> list[int]:
        """Get 1-based indices for backend names.

        Args:
            backend_names: List of backend keys.

        Returns:
            List of 1-based indices.
        """
        backends_path = str(self.inference_dir / "backends.toml")
        return get_backend_indices_helper(backends_path, backend_names)

    def verify_backends_enabled(self, expected_enabled: list[str]) -> None:
        """Verify backends.toml has expected backends enabled.

        Args:
            expected_enabled: List of backend keys expected to be enabled.
        """
        backends_path = str(self.inference_dir / "backends.toml")
        verify_backends_toml(backends_path, expected_enabled)

    def verify_routing(
        self,
        expected_active: str,
        expected_default: str | None = None,
        expected_fallback_order: list[str] | None = None,
    ) -> None:
        """Verify routing profile configuration.

        Args:
            expected_active: Expected active profile name.
            expected_default: Expected default backend (optional).
            expected_fallback_order: Expected fallback order (optional).
        """
        routing_path = str(self.inference_dir / "routing_profiles.toml")
        verify_routing_profile(routing_path, expected_active, expected_default, expected_fallback_order)

    def verify_run_execution(self, expected: RunExecution | None) -> None:
        """Verify where runs execute, as pipelex.toml sets it.

        Args:
            expected: The expected `[run] execution`, or None when the file should set none.
        """
        actual = read_run_execution(pipelex_toml_path=self.pipelex_dir / "pipelex.toml")
        assert actual == expected, f"Expected [run] execution {expected!r}, got {actual!r}"

    def verify_telemetry(self, expected_mode: str) -> None:
        """Verify telemetry configuration.

        Args:
            expected_mode: Expected telemetry mode.
        """
        telemetry_path = str(self.pipelex_dir / "telemetry.toml")
        verify_telemetry_config(telemetry_path, expected_mode)

    def verify_file_exists(self, relative_path: str) -> None:
        """Verify a file exists in the pipelex directory.

        Args:
            relative_path: Path relative to .pipelex directory.
        """
        file_path = self.pipelex_dir / relative_path
        assert file_path.exists(), f"Expected file {file_path} to exist"

    def verify_file_not_exists(self, relative_path: str) -> None:
        """Verify a file does not exist in the pipelex directory.

        Args:
            relative_path: Path relative to .pipelex directory.
        """
        file_path = self.pipelex_dir / relative_path
        assert not file_path.exists(), f"Expected file {file_path} to not exist"