Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/ask for inference backends (#398)

Louis Choquelf760b4ef62ca
diff --git a/.pipelex/inference/backends.toml b/.pipelex/inference/backends.toml
index b4fbef7ce..f330e0513 100644
--- a/.pipelex/inference/backends.toml
+++ b/.pipelex/inference/backends.toml
@@ -1,28 +1,35 @@
[pipelex_inference]
+display_name = "⭐ Pipelex Inference"
+enabled = true
endpoint = "https://inference.pipelex.com/v1"
api_key = "${PIPELEX_INFERENCE_API_KEY}"
[azure_openai]
+display_name = "Azure OpenAI"
enabled = true
endpoint = "${AZURE_API_BASE}"
api_key = "${AZURE_API_KEY}"
api_version = "${AZURE_API_VERSION}"
[bedrock]
-enabled = false
+display_name = "AWS Bedrock"
+enabled = true
aws_region = "${AWS_REGION}"
[google]
-enabled = false
+display_name = "Google AI"
+enabled = true
api_key = "${GOOGLE_API_KEY}"
[vertexai]
-enabled = false
+display_name = "Google Vertex AI"
+enabled = true
gcp_project_id = "${GCP_PROJECT_ID}"
gcp_location = "${GCP_LOCATION}"
gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
[openai]
+display_name = "OpenAI"
enabled = true
api_key = "${OPENAI_API_KEY}"
@@ -32,25 +39,29 @@ api_key = "${ANTHROPIC_API_KEY}"
claude_4_tokens_limit = 8192
[mistral]
+display_name = "Mistral AI"
enabled = true
api_key = "${MISTRAL_API_KEY}"
[xai]
-enabled = false
+display_name = "xAI"
+enabled = true
endpoint = "https://api.x.ai/v1"
api_key = "${XAI_API_KEY}"
[ollama]
-enabled = false
+enabled = true
endpoint = "http://localhost:11434/v1"
[blackboxai]
-enabled = false
+display_name = "BlackBox AI"
+enabled = true
endpoint = "https://api.blackbox.ai/v1"
api_key = "${BLACKBOX_API_KEY}"
[fal]
-enabled = false
+display_name = "FAL"
+enabled = true
api_key = "${FAL_API_KEY}"
[internal] # software-only backend, runs internally, without AI
diff --git a/.pipelex/telemetry.toml b/.pipelex/telemetry.toml
index da28e66d3..327955340 100644
--- a/.pipelex/telemetry.toml
+++ b/.pipelex/telemetry.toml
@@ -1,4 +1,4 @@
-telemetry_mode = "anonymous" # Values: "off" | "anonymous" | "identified"
+telemetry_mode = "anonymous" # Values: "off" | "anonymous" | "identified"
host = "https://eu.i.posthog.com"
project_api_key = "phc_HPJnNKpIXh0SxNDYyTAyUtnq9KxNNZJWQszynsWVx4Y"
respect_dnt = true
diff --git a/docs/index.md b/docs/index.md
index 98e8156f6..6dbf4c317 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -10,22 +10,20 @@ title: "Open-source AI workflows"
```bash
pip install pipelex
-pipelex init config
+pipelex init
```
-## Set your API key
+## Set your API key(s)
```bash
# Linux/MacOS
export PIPELEX_INFERENCE_API_KEY=###
-
# Windows PowerShell
$env:PIPELEX_INFERENCE_API_KEY="###"
-
# Windows CMD
set PIPELEX_INFERENCE_API_KEY=###
-# Note: of course, Pipelex automatically loads environment variables from `.env` files, that works too.
+# Note: Pipelex automatically loads environment variables from `.env` files, that works too.
```
**Where to get an API key:**
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index bf657355b..997229975 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -7,19 +7,20 @@ from typer.core import TyperGroup
from typing_extensions import override
from pipelex.cli.commands.build_cmd import build_app
+from pipelex.cli.commands.doctor_cmd import doctor_cmd
from pipelex.cli.commands.init_cmd import InitFocus, init_cmd
from pipelex.cli.commands.kit_cmd import kit_app
from pipelex.cli.commands.run_cmd import run_cmd
from pipelex.cli.commands.show_cmd import show_app
from pipelex.cli.commands.validate_cmd import validate_cmd
-from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract
+from pipelex.tools.misc.package_utils import get_package_version
class PipelexCLI(TyperGroup):
@override
def list_commands(self, ctx: Context) -> list[str]:
# List the commands in the proper order because natural ordering doesn't work between Typer groups and commands
- return ["init", "kit", "build", "validate", "run", "show"]
+ return ["init", "doctor", "kit", "build", "validate", "run", "show"]
@override
def get_command(self, ctx: Context, cmd_name: str) -> Command | None:
@@ -48,8 +49,9 @@ app = typer.Typer(
def app_callback(ctx: typer.Context) -> None:
"""Run pre-command checks like printing the logo and checking telemetry consent."""
console = Console()
+ package_version = get_package_version()
console.print(
- """
+ f"""
░█████████ ░[bold green4]██[/bold green4] ░██
░██ ░██ ░██
@@ -59,15 +61,14 @@ def app_callback(ctx: typer.Context) -> None:
░██ ░██░███ ░██ ░██ ░██ ░██ ░██ ░██
░██ ░██░██░█████ ░███████ ░██ ░███████ ░██ ░██
░██
- ░██
-
+ ░██ v{package_version}
"""
)
- # Skip checks if no command is being run (e.g., just --help) or if running init command
- if ctx.invoked_subcommand is None or ctx.invoked_subcommand == "init":
+ # Skip checks if no command is being run (e.g., just --help) or if running init/doctor command
+ if ctx.invoked_subcommand is None or ctx.invoked_subcommand in ("init", "doctor"):
return
- TelemetryManagerAbstract.telemetry_mode_just_set = init_cmd()
+ init_cmd()
@app.command(name="init", help="Initialize Pipelex configuration in a `.pipelex` directory")
@@ -79,6 +80,14 @@ def init_command(
init_cmd(focus=focus, reset=reset)
[email protected](name="doctor", help="Check Pipelex configuration health and suggest fixes")
+def doctor_command(
+ fix: Annotated[bool, typer.Option("--fix", "-f", help="Offer to fix detected issues interactively")] = False,
+) -> None:
+ """Check Pipelex configuration health."""
+ doctor_cmd(fix=fix)
+
+
app.add_typer(kit_app, name="kit", help="Manage kit assets: agent rules, migration rules")
app.add_typer(
build_app, name="build", help="Generate AI workflows from natural language requirements: pipelines in .plx format and python code to run them"
diff --git a/pipelex/cli/commands/doctor_cmd.py b/pipelex/cli/commands/doctor_cmd.py
new file mode 100644
index 000000000..874f1f295
--- /dev/null
+++ b/pipelex/cli/commands/doctor_cmd.py
@@ -0,0 +1,427 @@
+"""Doctor command for checking Pipelex configuration health."""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import sys
+
+from pydantic import ValidationError
+from rich.console import Console
+from rich.panel import Panel
+from rich.prompt import Confirm
+from rich.text import Text
+
+from pipelex.cli.commands.init_cmd import InitFocus, init_cmd, init_config
+from pipelex.cogt.model_backends.backend_library import BackendCredentialsReport
+from pipelex.config import PipelexConfig
+from pipelex.core.validation import report_validation_error
+from pipelex.system.configuration.config_loader import config_manager
+from pipelex.system.environment import get_optional_env
+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryConfig
+from pipelex.tools.misc.dict_utils import extract_vars_from_strings_recursive
+from pipelex.tools.misc.file_utils import path_exists
+from pipelex.tools.misc.placeholder import value_is_placeholder
+from pipelex.tools.misc.toml_utils import load_toml_from_path
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+
+
+def check_config_files() -> tuple[bool, int, str]:
+ """Check if configuration files are present and main config is valid.
+
+ Returns:
+ Tuple of (is_healthy, missing_count, message)
+ """
+ # Check for missing files
+ try:
+ missing_count = init_config(reset=False, dry_run=True)
+ except Exception as exc:
+ return False, 0, f"Error checking config files: {exc}"
+
+ # Check if main config can be loaded using the hub's setup
+ pipelex_config_path = ".pipelex/pipelex.toml"
+ if path_exists(pipelex_config_path):
+ try:
+ # Suppress stderr and stdout to prevent tracebacks from being printed
+ with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()):
+ config = config_manager.load_config()
+ PipelexConfig.model_validate(config)
+ except ValidationError as validation_error:
+ validation_error_msg = report_validation_error(category="config", validation_error=validation_error)
+ msg = f"Configuration validation failed:\n{validation_error_msg}"
+ return False, 0, msg
+ except Exception as exc:
+ return False, 0, f"Error loading pipelex.toml: {exc}"
+
+ # Report results
+ if missing_count == 0:
+ return True, 0, "All configuration files present and valid"
+ return False, missing_count, f"{missing_count} configuration file(s) missing"
+
+
+def check_telemetry_config() -> tuple[bool, str]:
+ """Check if telemetry configuration is valid.
+
+ Returns:
+ Tuple of (is_healthy, message)
+ """
+ # Use hard-coded path to avoid needing Pipelex initialization
+ telemetry_config_path = f".pipelex/{TELEMETRY_CONFIG_FILE_NAME}"
+
+ if not path_exists(telemetry_config_path):
+ return False, "Telemetry configuration file not found"
+
+ try:
+ toml_doc = load_toml_from_path(telemetry_config_path)
+ telemetry_config = TelemetryConfig.model_validate(toml_doc)
+ return True, f"Telemetry configured (mode: {telemetry_config.telemetry_mode})"
+ except ValidationError as validation_error:
+ validation_error_msg = format_pydantic_validation_error(validation_error)
+ msg = f"Invalid telemetry configuration: {validation_error_msg}"
+ return False, msg
+ except Exception as exc:
+ return False, f"Error loading telemetry config: {exc}"
+
+
+def check_backend_credentials() -> tuple[bool, dict[str, BackendCredentialsReport], str]:
+ """Check if backend credentials are properly configured.
+
+ Returns:
+ Tuple of (is_healthy, backend_reports_dict, summary_message)
+ """
+ # Use hard-coded path to avoid needing Pipelex initialization
+ backends_toml_path = ".pipelex/inference/backends.toml"
+
+ if not path_exists(backends_toml_path):
+ return False, {}, "Backend configuration file not found"
+
+ try:
+ backends_dict = load_toml_from_path(backends_toml_path)
+ backend_reports: dict[str, BackendCredentialsReport] = {}
+ all_backends_valid = True
+
+ for backend_name, backend_dict in backends_dict.items():
+ # Skip internal backend
+ if backend_name == "internal":
+ continue
+
+ # Only check enabled backends
+ if isinstance(backend_dict, dict):
+ enabled = backend_dict.get("enabled", True) # type: ignore[union-attr]
+ else:
+ enabled = True
+ if not enabled:
+ continue
+
+ # Extract all variable placeholders from the backend config
+ required_vars_set = extract_vars_from_strings_recursive(backend_dict)
+ required_vars = sorted(required_vars_set)
+
+ # Check status of each variable
+ missing_vars: list[str] = []
+ placeholder_vars: list[str] = []
+
+ for var_name in required_vars:
+ var_value = get_optional_env(var_name)
+ if var_value is None:
+ missing_vars.append(var_name)
+ elif value_is_placeholder(var_value):
+ placeholder_vars.append(var_name)
+
+ # Determine if all credentials are valid for this backend
+ backend_valid = len(missing_vars) == 0 and len(placeholder_vars) == 0
+
+ # Create report for this backend
+ backend_report = BackendCredentialsReport(
+ backend_name=backend_name,
+ required_vars=required_vars,
+ missing_vars=missing_vars,
+ placeholder_vars=placeholder_vars,
+ all_credentials_valid=backend_valid,
+ )
+ backend_reports[backend_name] = backend_report
+
+ if not backend_valid:
+ all_backends_valid = False
+
+ if all_backends_valid:
+ backend_count = len(backend_reports)
+ return True, backend_reports, f"All {backend_count} enabled backend(s) have valid credentials"
+
+ # Count backends with issues
+ backends_with_issues = sum(1 for r in backend_reports.values() if not r.all_credentials_valid)
+ return False, backend_reports, f"{backends_with_issues} backend(s) have missing or invalid credentials"
+
+ except Exception as exc:
+ return False, {}, f"Error checking backend credentials: {exc}"
+
+
+def display_health_report(
+ console: Console,
+ config_healthy: bool,
+ config_message: str,
+ config_missing_count: int,
+ telemetry_healthy: bool,
+ telemetry_message: str,
+ backends_healthy: bool,
+ backends_message: str,
+ backend_reports: dict[str, BackendCredentialsReport],
+) -> None:
+ """Display a comprehensive health report.
+
+ Args:
+ console: Rich Console instance for output
+ config_healthy: Whether config files check passed
+ config_message: Message about config files status
+ config_missing_count: Number of missing config files
+ telemetry_healthy: Whether telemetry check passed
+ telemetry_message: Message about telemetry status
+ backends_healthy: Whether backends check passed
+ backends_message: Message about backends status
+ backend_reports: Dict of backend credential reports
+ """
+ all_healthy = config_healthy and telemetry_healthy and backends_healthy
+
+ # Overall status panel
+ if all_healthy:
+ status_text = Text("Overall Status: ✅ All systems healthy", style="bold green")
+ else:
+ status_text = Text("Overall Status: ⚠️ Issues Found", style="bold yellow")
+
+ status_panel = Panel(
+ status_text,
+ title="[bold cyan]Pipelex Health Check[/bold cyan]",
+ border_style="cyan" if all_healthy else "yellow",
+ padding=(1, 2),
+ )
+ console.print()
+ console.print(status_panel)
+ console.print()
+
+ # Configuration Files section
+ console.print("[bold]Configuration Files[/bold]")
+ if config_healthy:
+ console.print(f" [green]✓[/green] {config_message}")
+ else:
+ console.print(f" [red]✗[/red] {config_message}")
+ console.print()
+
+ # Telemetry Configuration section
+ console.print("[bold]Telemetry Configuration[/bold]")
+ if telemetry_healthy:
+ console.print(f" [green]✓[/green] {telemetry_message}")
+ else:
+ console.print(f" [red]✗[/red] {telemetry_message}")
+ console.print()
+
+ # Backend Credentials section
+ console.print("[bold]Backend Credentials[/bold]")
+ if backends_healthy:
+ console.print(f" [green]✓[/green] {backends_message}")
+ elif not backend_reports:
+ # No backends were checked (e.g., file not found)
+ console.print(f" [red]✗[/red] {backends_message}")
+ else:
+ console.print(f" [yellow]⚠[/yellow] {backends_message}")
+ console.print()
+
+ # Show details for each backend
+ for backend_name, backend_report in backend_reports.items():
+ if backend_report.all_credentials_valid:
+ console.print(f" [dim]{backend_name}[/dim]")
+ console.print(" [green]✓[/green] All credentials set")
+ else:
+ console.print(f" [bold]{backend_name}[/bold]")
+ if backend_report.missing_vars:
+ console.print(f" [red]✗[/red] Missing: {', '.join(backend_report.missing_vars)}")
+ if backend_report.placeholder_vars:
+ console.print(f" [yellow]⚠[/yellow] Placeholders: {', '.join(backend_report.placeholder_vars)}")
+ console.print()
+
+ # Recommended actions
+ if not all_healthy:
+ console.print("[bold]Recommended Actions[/bold]")
+
+ # Check what can be auto-fixed
+ can_auto_fix_config = not config_healthy and config_missing_count > 0
+ can_auto_fix_telemetry = not telemetry_healthy and "not found" in telemetry_message.lower()
+
+ if can_auto_fix_config:
+ console.print(" • Run [cyan]pipelex init config[/cyan] to install missing configuration files")
+
+ if can_auto_fix_telemetry:
+ console.print(" • Run [cyan]pipelex init telemetry[/cyan] to configure telemetry preferences")
+
+ if not backends_healthy and backend_reports:
+ # Collect all missing and placeholder vars
+ all_missing_vars: set[str] = set()
+ all_placeholder_vars: set[str] = set()
+
+ for backend_report in backend_reports.values():
+ if not backend_report.all_credentials_valid:
+ all_missing_vars.update(backend_report.missing_vars)
+ all_placeholder_vars.update(backend_report.placeholder_vars)
+
+ if all_missing_vars:
+ console.print(" • Set the following environment variables:")
+ for var_name in sorted(all_missing_vars):
+ console.print(f" - {var_name}")
+
+ if all_placeholder_vars:
+ console.print(" • Replace placeholder values for:")
+ for var_name in sorted(all_placeholder_vars):
+ console.print(f" - {var_name}")
+
+ console.print()
+
+ # Only suggest --fix if there are auto-fixable issues
+ if can_auto_fix_config or can_auto_fix_telemetry:
+ console.print("[dim]Run[/dim] [cyan]pipelex doctor --fix[/cyan] [dim]to interactively fix auto-fixable issues.[/dim]")
+ console.print()
+
+ # Show Discord support for manual-fix issues (regardless of --fix flag)
+ has_config_validation_error = not config_healthy and config_missing_count == 0
+ has_backend_credential_issues = not backends_healthy and backend_reports
+ if has_config_validation_error or has_backend_credential_issues:
+ console.print("[dim]If you need help with manual fixes, join our Discord community:[/dim]")
+ console.print("[cyan]https://go.pipelex.com/discord[/cyan]")
+ console.print()
+
+
+def doctor_cmd(
+ fix: bool = False,
+) -> None:
+ """Check Pipelex configuration health and suggest fixes.
+
+ Args:
+ fix: If True, offer to fix detected issues interactively
+ """
+ console = Console()
+
+ try:
+ # Run health checks
+ config_healthy, config_missing_count, config_message = check_config_files()
+ telemetry_healthy, telemetry_message = check_telemetry_config()
+ backends_healthy, backend_reports, backends_message = check_backend_credentials()
+
+ # Display report
+ display_health_report(
+ console=console,
+ config_healthy=config_healthy,
+ config_message=config_message,
+ config_missing_count=config_missing_count,
+ telemetry_healthy=telemetry_healthy,
+ telemetry_message=telemetry_message,
+ backends_healthy=backends_healthy,
+ backends_message=backends_message,
+ backend_reports=backend_reports,
+ )
+
+ all_healthy = config_healthy and telemetry_healthy and backends_healthy
+
+ # Exit code: 0 if healthy, 1 if issues found
+ if all_healthy:
+ sys.exit(0)
+
+ # Determine what can be auto-fixed
+ can_fix_config = not config_healthy and config_missing_count > 0
+ can_fix_telemetry = not telemetry_healthy and "not found" in telemetry_message.lower()
+ has_auto_fixable_issues = can_fix_config or can_fix_telemetry
+
+ # Determine what requires manual fixes
+ has_config_validation_error = not config_healthy and config_missing_count == 0
+ has_backend_credential_issues = not backends_healthy and backend_reports
+
+ # If --fix flag is provided, offer to fix auto-fixable issues
+ if fix and has_auto_fixable_issues:
+ console.print("[bold yellow]Interactive Fix Mode[/bold yellow]")
+ console.print()
+
+ # Fix missing config files
+ if can_fix_config:
+ if Confirm.ask(f"[bold]Install {config_missing_count} missing configuration file(s)?[/bold]", default=True):
+ try:
+ console.print()
+ init_cmd(focus=InitFocus.CONFIG, reset=False, skip_confirmation=True)
+ console.print("[green]✓[/green] Configuration files installed")
+ except Exception as exc:
+ console.print(f"[red]Failed to install configuration files: {exc!s}[/red]")
+ console.print()
+
+ # Fix missing telemetry config
+ if can_fix_telemetry:
+ if Confirm.ask("[bold]Configure telemetry preferences?[/bold]", default=True):
+ try:
+ console.print()
+ init_cmd(focus=InitFocus.TELEMETRY, reset=False, skip_confirmation=True)
+ console.print("[green]✓[/green] Telemetry configured")
+ except Exception as exc:
+ console.print(f"[red]Failed to configure telemetry: {exc!s}[/red]")
+ console.print()
+
+ # Handle issues that can't be auto-fixed
+ if has_config_validation_error or has_backend_credential_issues:
+ console.print("[bold yellow]Manual Fixes Required[/bold yellow]")
+ console.print()
+
+ # Config validation errors
+ if has_config_validation_error:
+ console.print("[bold]Configuration validation error:[/bold]")
+ console.print(f" {config_message}")
+ console.print()
+ console.print("You can fix this manually by editing [cyan].pipelex/pipelex.toml[/cyan]")
+ console.print("or run [cyan]pipelex init config --reset[/cyan] to regenerate from template.")
+ console.print()
+
+ # Backend credentials
+ if has_backend_credential_issues:
+ all_missing_vars: set[str] = set()
+ for backend_report in backend_reports.values():
+ if not backend_report.all_credentials_valid:
+ all_missing_vars.update(backend_report.missing_vars)
+
+ if all_missing_vars:
+ console.print("[bold]Backend credentials:[/bold]")
+ console.print()
+ console.print("Set the following environment variables:")
+ console.print()
+
+ # Show .env file syntax first
+ console.print("[dim]# In your .env file:[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f"{var_name}=[yellow]your_value_here[/yellow]")
+ console.print()
+
+ # Show shell syntax for different platforms
+ console.print("[dim]# Or in your shell:[/dim]")
+ console.print()
+
+ # Linux/MacOS
+ console.print("[dim]# Linux/MacOS[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f"export {var_name}=[yellow]your_value_here[/yellow]")
+ console.print()
+
+ # Windows PowerShell
+ console.print("[dim]# Windows PowerShell[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f'$env:{var_name}="[yellow]your_value_here[/yellow]"')
+ console.print()
+
+ # Windows CMD
+ console.print("[dim]# Windows CMD[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f"set {var_name}=[yellow]your_value_here[/yellow]")
+ console.print()
+
+ sys.exit(1)
+
+ except Exception as exc:
+ # Handle unexpected errors gracefully without printing traces
+ console.print()
+ console.print(f"[red]✗ Unexpected error: {exc!s}[/red]")
+ console.print()
+ console.print("[dim]If you need help, join our Discord community:[/dim]")
+ console.print("[cyan]https://go.pipelex.com/discord[/cyan]")
+ console.print()
+ sys.exit(1)
diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.py
index 3c597034e..63312092c 100644
--- a/pipelex/cli/commands/init_cmd.py
+++ b/pipelex/cli/commands/init_cmd.py
@@ -1,32 +1,86 @@
import os
import shutil
-from importlib.metadata import metadata
+from typing import Any
import typer
-from rich.console import Console, Group
-from rich.panel import Panel
-from rich.prompt import Confirm, Prompt
-from rich.table import Table
-from rich.text import Text
-
+from rich.console import Console
+from rich.prompt import Confirm
+
+from pipelex.cli.commands.init_ui import (
+ InitFocus,
+ build_backend_selection_panel,
+ build_initialization_panel,
+ build_telemetry_selection_panel,
+ display_already_configured_message,
+ display_selected_backends,
+ get_backend_options_from_toml,
+ get_currently_enabled_backends,
+ prompt_backend_indices,
+ prompt_telemetry_mode,
+)
from pipelex.exceptions import PipelexCLIError
from pipelex.kit.paths import get_configs_dir
from pipelex.system.configuration.config_loader import config_manager
from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryMode
+from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract
from pipelex.tools.misc.file_utils import path_exists
from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
-from pipelex.types import StrEnum
-PACKAGE_NAME = __name__.split(".", maxsplit=1)[0]
-PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]
+def update_backends_in_toml(toml_doc: Any, selected_indices: list[int], backend_options: list[tuple[str, str]]) -> None:
+ """Update the backends.toml document with selected backends.
+
+ Args:
+ toml_doc: The TOML document to update.
+ selected_indices: List of backend indices to enable.
+ backend_options: List of available backend options.
+ """
+ selected_backend_keys = {backend_options[idx][0] for idx in selected_indices}
+
+ # Disable all backends first (except internal)
+ for backend_key in toml_doc:
+ if backend_key != "internal" and backend_key in toml_doc:
+ backend_section = toml_doc[backend_key]
+ # Set enabled field based on selection (works with tomlkit's special types)
+ backend_section["enabled"] = backend_key in selected_backend_keys # type: ignore[index]
+
+
+def customize_backends_config() -> None:
+ """Interactively customize which inference backends are enabled in backends.toml."""
+ console = Console()
+ backends_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "backends.toml")
+ template_backends_path = os.path.join(str(get_configs_dir()), "inference", "backends.toml")
+
+ if not path_exists(backends_toml_path):
+ console.print("[yellow]⚠ Warning: backends.toml not found, skipping backend customization[/yellow]")
+ return
+
+ try:
+ # Get backend options from template and existing config
+ existing_path = backends_toml_path if path_exists(backends_toml_path) else None
+ backend_options = get_backend_options_from_toml(template_backends_path, existing_path)
+
+ # Get currently enabled backends to show user their current selection
+ currently_enabled = get_currently_enabled_backends(backends_toml_path, backend_options)
+
+ # Load the backends.toml file
+ toml_doc = load_toml_with_tomlkit(backends_toml_path)
+ console.print()
+
+ # UI: Display panel and get user selection
+ console.print(build_backend_selection_panel(backend_options, currently_enabled))
+ selected_indices = prompt_backend_indices(console, backend_options, currently_enabled)
-class InitFocus(StrEnum):
- """Focus options for initialization."""
+ # Business logic: Update TOML
+ update_backends_in_toml(toml_doc, selected_indices, backend_options)
+ save_toml_to_path(toml_doc, backends_toml_path)
- ALL = "all"
- CONFIG = "config"
- TELEMETRY = "telemetry"
+ # UI: Display confirmation
+ display_selected_backends(console, selected_indices, backend_options)
+
+ except Exception as exc:
+ console.print(f"[yellow]⚠ Warning: Failed to customize backends: {exc}[/yellow]")
+ console.print("[dim]You can manually edit .pipelex/inference/backends.toml later[/dim]")
def init_config(reset: bool = False, dry_run: bool = False) -> int:
@@ -60,14 +114,15 @@ def init_config(reset: bool = False, dry_run: bool = False) -> int:
continue
if os.path.isdir(src_item):
- os.makedirs(dst_item, exist_ok=True)
- copy_directory_structure(src_item, dst_item, relative_item)
+ if not dry_run:
+ os.makedirs(dst_item, exist_ok=True)
+ copy_directory_structure(src_item, dst_item, relative_item, dry_run)
elif os.path.exists(dst_item) and not reset:
existing_files.append(relative_item)
else:
- shutil.copy2(src_item, dst_item)
if not dry_run:
- copied_files.append(relative_item)
+ shutil.copy2(src_item, dst_item)
+ copied_files.append(relative_item)
copy_directory_structure(src_dir=config_template_dir, dst_dir=target_config_dir, dry_run=dry_run)
@@ -95,204 +150,274 @@ def init_config(reset: bool = False, dry_run: bool = False) -> int:
return len(copied_files)
+def setup_telemetry(console: Console, telemetry_config_path: str) -> TelemetryMode:
+ """Set up telemetry configuration interactively.
+
+ Args:
+ console: Rich Console instance for user interaction.
+ telemetry_config_path: Path to save the telemetry configuration.
+
+ Returns:
+ The selected TelemetryMode.
+
+ Raises:
+ typer.Exit: If user chooses to quit.
+ """
+ console.print()
+ console.print(build_telemetry_selection_panel())
+
+ telemetry_mode = prompt_telemetry_mode(console)
+
+ # Save telemetry config
+ template_path = os.path.join(str(get_configs_dir()), TELEMETRY_CONFIG_FILE_NAME)
+ toml_doc = load_toml_with_tomlkit(template_path)
+ toml_doc["telemetry_mode"] = telemetry_mode
+ save_toml_to_path(toml_doc, telemetry_config_path)
+
+ console.print(f"\n[green]✓[/green] Telemetry mode set to: [bold cyan]{telemetry_mode}[/bold cyan]")
+
+ return telemetry_mode
+
+
+def determine_needs(
+ reset: bool,
+ check_config: bool,
+ check_inference: bool,
+ check_telemetry: bool,
+ backends_toml_path: str,
+ telemetry_config_path: str,
+) -> tuple[bool, bool, bool]:
+ """Determine what needs to be initialized based on current state.
+
+ Args:
+ reset: Whether this is a reset operation.
+ check_config: Whether to check config files.
+ check_inference: Whether to check inference setup.
+ check_telemetry: Whether to check telemetry setup.
+ backends_toml_path: Path to backends.toml file.
+ telemetry_config_path: Path to telemetry config file.
+
+ Returns:
+ Tuple of (needs_config, needs_inference, needs_telemetry) booleans.
+ """
+ nb_missing_config_files = init_config(reset=False, dry_run=True) if check_config else 0
+ needs_config = check_config and (nb_missing_config_files > 0 or reset)
+ needs_inference = check_inference and (not path_exists(backends_toml_path) or reset)
+ needs_telemetry = check_telemetry and (not path_exists(telemetry_config_path) or reset)
+
+ return needs_config, needs_inference, needs_telemetry
+
+
+def handle_already_configured(
+ focus: InitFocus,
+ console: Console,
+ backends_toml_path: str,
+ telemetry_config_path: str,
+) -> bool:
+ """Handle the case when everything is already configured.
+
+ Args:
+ focus: The initialization focus area.
+ console: Rich Console instance for output.
+ backends_toml_path: Path to backends.toml file.
+ telemetry_config_path: Path to telemetry config file.
+
+ Returns:
+ True if user wants to reconfigure, False otherwise.
+ """
+ # Map focus to config path for display
+ config_path_map = {
+ InitFocus.INFERENCE: backends_toml_path,
+ InitFocus.TELEMETRY: telemetry_config_path,
+ InitFocus.CONFIG: ".pipelex/",
+ }
+
+ config_path = config_path_map.get(focus, "")
+ return display_already_configured_message(focus, console, config_path)
+
+
+def update_needs_for_reconfigure(focus: InitFocus) -> tuple[bool, bool, bool]:
+ """Update needs flags when user wants to reconfigure.
+
+ Args:
+ focus: The initialization focus area.
+
+ Returns:
+ Tuple of (needs_config, needs_inference, needs_telemetry) booleans.
+ """
+ needs_config = focus == InitFocus.CONFIG
+ needs_inference = focus == InitFocus.INFERENCE
+ needs_telemetry = focus == InitFocus.TELEMETRY
+
+ return needs_config, needs_inference, needs_telemetry
+
+
+def confirm_initialization(
+ console: Console,
+ needs_config: bool,
+ needs_inference: bool,
+ needs_telemetry: bool,
+ reset: bool,
+ focus: InitFocus,
+) -> bool:
+ """Ask user to confirm initialization.
+
+ Args:
+ console: Rich Console instance for user interaction.
+ needs_config: Whether config initialization is needed.
+ needs_inference: Whether inference setup is needed.
+ needs_telemetry: Whether telemetry setup is needed.
+ reset: Whether this is a reset operation.
+ focus: The initialization focus area.
+
+ Returns:
+ True if user confirms, False otherwise.
+
+ Raises:
+ typer.Exit: If user cancels initialization.
+ """
+ console.print()
+ console.print(build_initialization_panel(needs_config, needs_inference, needs_telemetry, reset))
+
+ if not Confirm.ask("[bold]Continue with initialization?[/bold]", default=True):
+ console.print("\n[yellow]Initialization cancelled.[/yellow]")
+ if needs_config or needs_inference or needs_telemetry:
+ match focus:
+ case InitFocus.ALL:
+ init_cmd_str = "pipelex init"
+ case InitFocus.CONFIG | InitFocus.INFERENCE | InitFocus.TELEMETRY:
+ init_cmd_str = f"pipelex init {focus}"
+ console.print(f"[dim]You can initialize later by running:[/dim] [cyan]{init_cmd_str}[/cyan]")
+ console.print()
+ raise typer.Exit(code=0)
+
+ return True
+
+
+def execute_initialization(
+ console: Console,
+ needs_config: bool,
+ needs_inference: bool,
+ needs_telemetry: bool,
+ reset: bool,
+ check_inference: bool,
+ backends_toml_path: str,
+ telemetry_config_path: str,
+):
+ """Execute the initialization steps.
+
+ Args:
+ console: Rich Console instance for output.
+ needs_config: Whether to initialize config files.
+ needs_inference: Whether to set up inference backends.
+ needs_telemetry: Whether to set up telemetry.
+ reset: Whether this is a reset operation.
+ check_inference: Whether inference was in focus.
+ backends_toml_path: Path to backends.toml file.
+ telemetry_config_path: Path to telemetry config file.
+
+ """
+ # Step 1: Initialize config if needed
+ if needs_config:
+ # Check if backends.toml exists before copying
+ backends_existed_before = path_exists(backends_toml_path)
+
+ console.print()
+ init_config(reset=reset)
+
+ # If backends.toml was just created (freshly copied), always prompt for backend selection
+ backends_exists_now = path_exists(backends_toml_path)
+ backends_just_copied = not backends_existed_before and backends_exists_now
+
+ if backends_just_copied or (check_inference and backends_exists_now):
+ needs_inference = True
+
+ # Step 2: Set up inference backends if needed
+ if needs_inference:
+ console.print()
+ customize_backends_config()
+
+ # Step 3: Set up telemetry if needed
+ if needs_telemetry:
+ telemetry_mode = setup_telemetry(console, telemetry_config_path)
+ TelemetryManagerAbstract.telemetry_mode_just_set = telemetry_mode
+
+ console.print()
+
+
def init_cmd(
focus: InitFocus = InitFocus.ALL,
reset: bool = False,
-) -> TelemetryMode | None:
- """Initialize Pipelex configuration and telemetry if needed, in a unified flow.
+ skip_confirmation: bool = False,
+):
+ """Initialize Pipelex configuration, inference backends, and telemetry if needed, in a unified flow.
Args:
- focus: What to initialize - 'config', 'telemetry', or 'all' (default)
+ focus: What to initialize - 'config', 'inference', 'telemetry', or 'all' (default)
reset: Whether to reset/overwrite existing files
+ skip_confirmation: If True, skip the confirmation prompt (used when called from doctor --fix)
"""
console = Console()
pipelex_config_dir = config_manager.pipelex_config_dir
telemetry_config_path = os.path.join(pipelex_config_dir, TELEMETRY_CONFIG_FILE_NAME)
+ backends_toml_path = os.path.join(pipelex_config_dir, "inference", "backends.toml")
# Determine what to check based on focus parameter
check_config = focus in (InitFocus.ALL, InitFocus.CONFIG)
+ check_inference = focus in (InitFocus.ALL, InitFocus.INFERENCE)
check_telemetry = focus in (InitFocus.ALL, InitFocus.TELEMETRY)
# Check what needs to be initialized
- nb_missing_config_files = init_config(reset=False, dry_run=True) if check_config else 0
- needs_config = check_config and (nb_missing_config_files > 0 or reset)
- needs_telemetry = check_telemetry and (not path_exists(telemetry_config_path) or reset)
+ needs_config, needs_inference, needs_telemetry = determine_needs(
+ reset=reset,
+ check_config=check_config,
+ check_inference=check_inference,
+ check_telemetry=check_telemetry,
+ backends_toml_path=backends_toml_path,
+ telemetry_config_path=telemetry_config_path,
+ )
# Track if user already confirmed to avoid double prompting
user_already_confirmed = False
# If nothing needs to be done, handle based on focus
- if not needs_config and not needs_telemetry:
- match focus:
- case InitFocus.TELEMETRY:
- # Special case: if user explicitly asked for telemetry, offer to reconfigure
- console.print()
- console.print("[green]✓[/green] Telemetry preferences are already configured!")
- console.print()
- console.print(f"[dim]Configuration file:[/dim] [cyan]{telemetry_config_path}[/cyan]")
- console.print()
-
- if Confirm.ask("[bold]Would you like to reconfigure telemetry preferences?[/bold]", default=False):
- # User wants to reconfigure, so proceed with telemetry setup
- needs_telemetry = True
- user_already_confirmed = True
- else:
- console.print("\n[dim]No changes made.[/dim]")
- console.print()
- return None
-
- case InitFocus.ALL:
- console.print()
- console.print("[green]✓[/green] Pipelex is already fully initialized!")
- console.print()
- console.print("[dim]Configuration files are in place:[/dim] [cyan].pipelex/[/cyan]")
- console.print("[dim]Telemetry preferences are configured[/dim]")
- console.print()
- console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
- console.print(" [cyan]pipelex init --reset[/cyan]")
- console.print()
- return None
-
- case InitFocus.CONFIG:
- console.print()
- console.print("[green]✓[/green] Configuration files are already in place!")
- console.print()
- console.print("[dim]Configuration directory:[/dim] [cyan].pipelex/[/cyan]")
- console.print()
- console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
- console.print(f" [cyan]pipelex init {focus} --reset[/cyan]")
- console.print()
- return None
-
- try:
- # Show unified initialization prompt (skip if user already confirmed)
- if not user_already_confirmed:
+ if not needs_config and not needs_inference and not needs_telemetry:
+ if handle_already_configured(focus, console, backends_toml_path, telemetry_config_path):
+ # User wants to reconfigure
+ needs_config, needs_inference, needs_telemetry = update_needs_for_reconfigure(focus)
+ user_already_confirmed = True
+ else:
+ # User doesn't want to reconfigure, exit
+ console.print("\n[dim]No changes made.[/dim]")
console.print()
+ return
- # Build message based on what's being initialized
- message_parts: list[str] = []
- if reset:
- if needs_config and needs_telemetry:
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] telemetry preferences")
- elif needs_config:
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")
- elif needs_telemetry:
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] telemetry preferences")
- elif needs_config and needs_telemetry:
- message_parts.append("• Create required configuration files in [cyan].pipelex/[/cyan]")
- message_parts.append("• Ask you to choose your telemetry preferences")
- elif needs_config:
- message_parts.append("• Create required configuration files in [cyan].pipelex/[/cyan]")
- elif needs_telemetry:
- message_parts.append("• Ask you to choose your telemetry preferences")
-
- if needs_config and needs_telemetry:
- title_text = "[bold yellow]Resetting Configuration[/bold yellow]" if reset else "[bold cyan]Pipelex Initialization[/bold cyan]"
- elif needs_config:
- title_text = "[bold yellow]Resetting Configuration Files[/bold yellow]" if reset else "[bold cyan]Configuration Setup[/bold cyan]"
- else:
- title_text = "[bold yellow]Resetting Telemetry[/bold yellow]" if reset else "[bold cyan]Telemetry Setup[/bold cyan]"
-
- message = "\n".join(message_parts)
- border_color = "yellow" if reset else "cyan"
-
- panel = Panel(
- message,
- title=title_text,
- border_style=border_color,
- padding=(1, 2),
+ try:
+ # Show unified initialization prompt (skip if user already confirmed or skip_confirmation is True)
+ if not user_already_confirmed and not skip_confirmation:
+ confirm_initialization(
+ console=console,
+ needs_config=needs_config,
+ needs_inference=needs_inference,
+ needs_telemetry=needs_telemetry,
+ reset=reset,
+ focus=focus,
)
- console.print(panel)
-
- if not Confirm.ask("[bold]Continue with initialization?[/bold]", default=True):
- console.print("\n[yellow]Initialization cancelled.[/yellow]")
- if needs_config:
- match focus:
- case InitFocus.ALL:
- init_cmd_str = "pipelex init"
- case InitFocus.CONFIG:
- init_cmd_str = f"pipelex init {focus}"
- case InitFocus.TELEMETRY:
- init_cmd_str = f"pipelex init {focus}"
- console.print(f"[dim]You can initialize later by running:[/dim] [cyan]{init_cmd_str}[/cyan]")
- console.print()
- raise typer.Exit(code=0)
else:
- # User already confirmed, just add a blank line for spacing
+ # User already confirmed or skip_confirmation is True, just add a blank line for spacing
console.print()
- # Step 1: Initialize config if needed
- if needs_config:
- console.print()
- init_config(reset=reset)
-
- # Step 2: Set up telemetry if needed
- telemetry_mode: TelemetryMode | None = None
- if needs_telemetry:
- console.print()
-
- # Create a table for telemetry options
- table = Table(show_header=False, box=None, padding=(0, 2))
- table.add_column(style="bold cyan", justify="right")
- table.add_column(style="bold")
- table.add_column()
-
- table.add_row("[1]", TelemetryMode.OFF, "No telemetry data collected")
- table.add_row("[2]", TelemetryMode.ANONYMOUS, "Anonymous usage data only")
- table.add_row("[3]", TelemetryMode.IDENTIFIED, "Usage data with user identification")
- table.add_row("[Q]", "[dim]quit[/dim]", "[dim]Exit without configuring[/dim]")
-
- description = Text(
- "Pipelex can collect anonymous usage data to help improve the product.",
- style="dim",
- )
- telemetry_panel = Panel(
- Group(description, Text(""), table),
- title="[bold yellow]Telemetry Configuration[/bold yellow]",
- border_style="yellow",
- padding=(1, 2),
- )
- console.print(telemetry_panel)
-
- # Map choice to telemetry mode
- mode_map: dict[str, TelemetryMode] = {
- "1": TelemetryMode.OFF,
- "2": TelemetryMode.ANONYMOUS,
- "3": TelemetryMode.IDENTIFIED,
- "off": TelemetryMode.OFF,
- "anonymous": TelemetryMode.ANONYMOUS,
- "identified": TelemetryMode.IDENTIFIED,
- }
-
- # Loop until valid input
- while telemetry_mode is None:
- choice_str = Prompt.ask("[bold]Enter your choice[/bold]", console=console)
- choice_normalized = choice_str.lower().strip()
-
- # Handle quit option
- if choice_normalized in ("q", "quit"):
- console.print("\n[yellow]Exiting without configuring telemetry.[/yellow]")
- raise typer.Exit(code=0)
-
- if choice_normalized in mode_map:
- telemetry_mode = mode_map[choice_normalized]
- else:
- console.print(
- f"[red]Invalid choice: '{choice_str}'.[/red] "
- "Please enter [cyan]1[/cyan], [cyan]2[/cyan], [cyan]3[/cyan], or [cyan]q[/cyan] to quit.\n"
- )
-
- # Save telemetry config
- template_path = os.path.join(str(get_configs_dir()), TELEMETRY_CONFIG_FILE_NAME)
- toml_doc = load_toml_with_tomlkit(template_path)
- toml_doc["telemetry_mode"] = telemetry_mode
- save_toml_to_path(toml_doc, telemetry_config_path)
-
- console.print(f"\n[green]✓[/green] Telemetry mode set to: [bold cyan]{telemetry_mode}[/bold cyan]")
-
- console.print()
- return telemetry_mode
+ # Execute initialization steps
+ execute_initialization(
+ console=console,
+ needs_config=needs_config,
+ needs_inference=needs_inference,
+ needs_telemetry=needs_telemetry,
+ reset=reset,
+ check_inference=check_inference,
+ backends_toml_path=backends_toml_path,
+ telemetry_config_path=telemetry_config_path,
+ )
except typer.Exit:
# Re-raise Exit exceptions
@@ -301,4 +426,4 @@ def init_cmd(
console.print(f"\n[red]⚠ Warning: Initialization failed: {exc}[/red]", style="bold")
if needs_config:
console.print("[red]Please run 'pipelex init config' manually.[/red]")
- return None
+ return
diff --git a/pipelex/cli/commands/init_ui.py b/pipelex/cli/commands/init_ui.py
new file mode 100644
index 000000000..309d15724
--- /dev/null
+++ b/pipelex/cli/commands/init_ui.py
@@ -0,0 +1,437 @@
+"""UI components for the init command.
+
+This module contains all user interface logic for the Pipelex initialization process,
+including prompts, panels, and user input validation.
+"""
+
+from __future__ import annotations
+
+import typer
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.prompt import Confirm, Prompt
+from rich.table import Table
+from rich.text import Text
+
+from pipelex.system.telemetry.telemetry_config import TelemetryMode
+from pipelex.tools.misc.file_utils import path_exists
+from pipelex.tools.misc.string_utils import snake_to_capitalize_first_letter
+from pipelex.tools.misc.toml_utils import load_toml_from_path
+from pipelex.types import StrEnum
+
+
+def get_backend_options_from_toml(template_path: str, existing_path: str | None = None) -> list[tuple[str, str]]:
+ """Get backend options dynamically from TOML files.
+
+ Args:
+ template_path: Path to the template backends.toml file.
+ existing_path: Optional path to existing user backends.toml file.
+
+ Returns:
+ List of tuples (backend_key, display_name).
+ """
+ backend_options: list[tuple[str, str]] = []
+ seen_backends: set[str] = set()
+
+ # Read template backends
+ if path_exists(template_path):
+ toml_doc = load_toml_from_path(template_path)
+ for backend_key in toml_doc:
+ if backend_key != "internal": # Skip internal backend
+ backend_section = toml_doc[backend_key]
+ # Try to get display_name from TOML, fallback to converted snake_case
+ if isinstance(backend_section, dict) and "display_name" in backend_section:
+ display_name = str(backend_section["display_name"]) # type: ignore[arg-type]
+ else:
+ display_name = snake_to_capitalize_first_letter(backend_key)
+ backend_options.append((backend_key, display_name))
+ seen_backends.add(backend_key)
+
+ # Add any additional backends from existing config (custom backends user may have added)
+ if existing_path and path_exists(existing_path):
+ toml_doc = load_toml_from_path(existing_path)
+ for backend_key in toml_doc:
+ if backend_key != "internal" and backend_key not in seen_backends:
+ backend_section = toml_doc[backend_key]
+ # Try to get display_name from TOML, fallback to converted snake_case
+ if isinstance(backend_section, dict) and "display_name" in backend_section:
+ display_name = str(backend_section["display_name"]) # type: ignore[arg-type]
+ else:
+ display_name = snake_to_capitalize_first_letter(backend_key)
+ backend_options.append((backend_key, display_name))
+ seen_backends.add(backend_key)
+
+ return backend_options
+
+
+def get_currently_enabled_backends(backends_toml_path: str, backend_options: list[tuple[str, str]]) -> list[int]:
+ """Get list of currently enabled backend indices from existing backends.toml.
+
+ Args:
+ backends_toml_path: Path to existing backends.toml file.
+ backend_options: List of backend options to match against.
+
+ Returns:
+ List of 0-based indices of currently enabled backends.
+ """
+ currently_enabled: list[int] = []
+
+ if not path_exists(backends_toml_path):
+ return currently_enabled
+
+ try:
+ toml_doc = load_toml_from_path(backends_toml_path)
+
+ # Create a mapping of backend_key to index
+ backend_key_to_index = {backend_key: idx for idx, (backend_key, _) in enumerate(backend_options)}
+
+ # Find which backends are currently enabled
+ for backend_key in toml_doc:
+ if backend_key != "internal" and backend_key in backend_key_to_index:
+ backend_section = toml_doc[backend_key]
+ if isinstance(backend_section, dict):
+ if backend_section.get("enabled", False) is True: # type: ignore[union-attr]
+ currently_enabled.append(backend_key_to_index[backend_key])
+
+ except Exception:
+ # If we can't read the file, just return empty list (silent failure is acceptable here)
+ return []
+
+ return sorted(currently_enabled)
+
+
+class InitFocus(StrEnum):
+ """Focus options for initialization."""
+
+ ALL = "all"
+ CONFIG = "config"
+ INFERENCE = "inference"
+ TELEMETRY = "telemetry"
+
+
+def build_backend_selection_panel(backend_options: list[tuple[str, str]], currently_enabled: list[int] | None = None) -> Panel:
+ """Create a Rich Panel for backend selection with options table.
+
+ Args:
+ backend_options: List of tuples (backend_key, display_name).
+ currently_enabled: Optional list of currently enabled backend indices (0-based).
+
+ Returns:
+ A Panel containing the backend selection interface.
+ """
+ table = Table(show_header=False, box=None, padding=(0, 2))
+ table.add_column(style="bold cyan", justify="right", width=4)
+ table.add_column(style="bold", width=30)
+ table.add_column(style="dim", width=15)
+
+ for idx, (_, backend_name) in enumerate(backend_options, start=1):
+ # Mark currently enabled backends
+ status = "[green]✓ enabled[/green]" if currently_enabled and (idx - 1) in currently_enabled else ""
+ table.add_row(f"[{idx}]", backend_name, status)
+
+ # Add special options at the end
+ table.add_row("[A]", "[dim]all - Select all backends[/dim]", "")
+ table.add_row("[Q]", "[dim]quit - Exit without configuring[/dim]", "")
+
+ # Update description based on whether we're showing current selection
+ if currently_enabled:
+ # Build current selection display with numbers and names
+ current_items: list[str] = []
+ for idx in sorted(currently_enabled):
+ backend_name = backend_options[idx][1]
+ current_items.append(f"{idx + 1} ({backend_name})")
+ current_selection = ", ".join(current_items)
+ description = Text(
+ f"Current selection: {current_selection}\n"
+ "Select which inference backends you have access to.\n"
+ "Enter numbers separated by commas or spaces (e.g., '1,5,6' or '1 5 6'), 'a' for all.\n"
+ "Press Enter to keep current selection.",
+ style="dim",
+ )
+ else:
+ description = Text(
+ "Select which inference backends you have access to.\n"
+ "Enter numbers separated by commas or spaces (e.g., '1,5,6' or '1 5 6'), 'a' for all.\n"
+ "Press Enter for the recommended default (1).",
+ style="dim",
+ )
+
+ return Panel(
+ Group(description, Text(""), table),
+ title="[bold yellow]Inference Backend Selection[/bold yellow]",
+ border_style="yellow",
+ padding=(1, 2),
+ )
+
+
+def prompt_backend_indices(console: Console, backend_options: list[tuple[str, str]], currently_enabled: list[int] | None = None) -> list[int]:
+ """Prompt user to select backend indices with validation.
+
+ Args:
+ console: Rich Console instance for user interaction.
+ backend_options: List of available backend options.
+ currently_enabled: Optional list of currently enabled backend indices (0-based).
+
+ Returns:
+ List of validated backend indices (0-based) selected by the user.
+
+ Raises:
+ typer.Exit: If user chooses to quit.
+ """
+ # Determine default based on current selection or fallback to first option
+ if currently_enabled:
+ default_indices = sorted(currently_enabled)
+ default_str = ",".join(str(i + 1) for i in default_indices)
+ else:
+ default_indices = [0]
+ default_str = "1"
+
+ selected_indices: list[int] = []
+ while True:
+ choice_str = Prompt.ask("[bold]Enter your choices[/bold]", default=default_str, console=console)
+ choice_input = choice_str.strip().lower()
+
+ # Handle quit option
+ if choice_input in ("q", "quit"):
+ console.print("\n[yellow]Exiting without configuring backends.[/yellow]")
+ raise typer.Exit(code=0)
+
+ # Handle all option
+ if choice_input in ("a", "all"):
+ selected_indices = list(range(len(backend_options)))
+ break
+
+ # Parse input - handle empty (use default)
+ if not choice_input:
+ selected_indices = default_indices
+ break
+
+ # Split by comma or space
+ parts = choice_input.replace(",", " ").split()
+
+ try:
+ # Parse as 1-based indices from user input
+ user_indices = [int(part.strip()) for part in parts if part.strip()]
+
+ # Validate all indices are in range (1-based)
+ invalid_indices = [i for i in user_indices if i < 1 or i > len(backend_options)]
+ if invalid_indices:
+ max_idx = len(backend_options)
+ console.print(
+ f"[red]Invalid choice(s): {invalid_indices}.[/red] "
+ f"Please enter numbers between 1 and {max_idx}, [cyan]a[/cyan] for all, or [cyan]q[/cyan] to quit.\n"
+ )
+ continue
+
+ # Convert to 0-based indices for internal use
+ selected_indices = [i - 1 for i in user_indices]
+ break
+
+ except ValueError:
+ console.print(
+ f"[red]Invalid input: '{choice_str}'.[/red] "
+ f"Please enter numbers separated by commas or spaces, [cyan]a[/cyan] for all, or [cyan]q[/cyan] to quit.\n"
+ )
+
+ return selected_indices
+
+
+def display_selected_backends(console: Console, selected_indices: list[int], backend_options: list[tuple[str, str]]) -> None:
+ """Display confirmation of selected backends.
+
+ Args:
+ console: Rich Console instance for output.
+ selected_indices: List of selected backend indices.
+ backend_options: List of available backend options.
+ """
+ selected_names = [backend_options[idx][1] for idx in sorted(selected_indices)]
+ console.print(f"\n[green]✓[/green] Configured {len(selected_names)} backend(s):")
+ for name in selected_names:
+ console.print(f" • {name}")
+
+
+def build_telemetry_selection_panel() -> Panel:
+ """Create a Rich Panel for telemetry mode selection.
+
+ Returns:
+ A Panel containing the telemetry selection interface.
+ """
+ table = Table(show_header=False, box=None, padding=(0, 2))
+ table.add_column(style="bold cyan", justify="right")
+ table.add_column(style="bold")
+ table.add_column()
+
+ table.add_row("[1]", TelemetryMode.OFF, "No telemetry data collected")
+ table.add_row("[2]", TelemetryMode.ANONYMOUS, "Anonymous usage data only")
+ table.add_row("[3]", TelemetryMode.IDENTIFIED, "Usage data with user identification")
+ table.add_row("[Q]", "[dim]quit[/dim]", "[dim]Exit without configuring[/dim]")
+
+ description = Text(
+ "Pipelex can collect anonymous usage data to help improve the product.",
+ style="dim",
+ )
+
+ return Panel(
+ Group(description, Text(""), table),
+ title="[bold yellow]Telemetry Configuration[/bold yellow]",
+ border_style="yellow",
+ padding=(1, 2),
+ )
+
+
+def prompt_telemetry_mode(console: Console) -> TelemetryMode:
+ """Prompt user to select telemetry mode with validation.
+
+ Args:
+ console: Rich Console instance for user interaction.
+
+ Returns:
+ Selected TelemetryMode.
+
+ Raises:
+ typer.Exit: If user chooses to quit.
+ """
+ # Map choice to telemetry mode
+ mode_map: dict[str, TelemetryMode] = {
+ "1": TelemetryMode.OFF,
+ "2": TelemetryMode.ANONYMOUS,
+ "3": TelemetryMode.IDENTIFIED,
+ "off": TelemetryMode.OFF,
+ "anonymous": TelemetryMode.ANONYMOUS,
+ "identified": TelemetryMode.IDENTIFIED,
+ }
+
+ # Loop until valid input
+ telemetry_mode: TelemetryMode | None = None
+ while telemetry_mode is None:
+ choice_str = Prompt.ask("[bold]Enter your choice[/bold]", console=console)
+ choice_input = choice_str.lower().strip()
+
+ # Handle quit option
+ if choice_input in ("q", "quit"):
+ console.print("\n[yellow]Exiting without configuring telemetry.[/yellow]")
+ raise typer.Exit(code=0)
+
+ if choice_input in mode_map:
+ telemetry_mode = mode_map[choice_input]
+ else:
+ console.print(
+ f"[red]Invalid choice: '{choice_str}'.[/red] "
+ "Please enter [cyan]1[/cyan], [cyan]2[/cyan], [cyan]3[/cyan], or [cyan]q[/cyan] to quit.\n"
+ )
+
+ return telemetry_mode
+
+
+def display_already_configured_message(focus: InitFocus, console: Console, config_path: str) -> bool:
+ """Display 'already configured' message and ask if user wants to reconfigure.
+
+ Args:
+ focus: The initialization focus area.
+ console: Rich Console instance for output.
+ config_path: Path to the configuration file.
+
+ Returns:
+ True if user wants to reconfigure, False otherwise.
+ """
+ # Mapping of focus to (subject, action_verb)
+ focus_messages = {
+ InitFocus.INFERENCE: ("Inference backends", "inference backends"),
+ InitFocus.TELEMETRY: ("Telemetry preferences", "telemetry preferences"),
+ InitFocus.CONFIG: ("Configuration files", "configuration"),
+ }
+
+ if focus == InitFocus.ALL:
+ console.print()
+ console.print("[green]✓[/green] Pipelex is already fully initialized!")
+ console.print()
+ console.print("[dim]Configuration files are in place:[/dim] [cyan].pipelex/[/cyan]")
+ console.print("[dim]Telemetry preferences are configured[/dim]")
+ console.print()
+ console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
+ console.print(" [cyan]pipelex init --reset[/cyan]")
+ console.print()
+ return False
+
+ if focus == InitFocus.CONFIG:
+ console.print()
+ console.print("[green]✓[/green] Configuration files are already in place!")
+ console.print()
+ console.print("[dim]Configuration directory:[/dim] [cyan].pipelex/[/cyan]")
+ console.print()
+ console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
+ console.print(f" [cyan]pipelex init {focus} --reset[/cyan]")
+ console.print()
+ return False
+
+ if focus in focus_messages:
+ subject, action_verb = focus_messages[focus]
+ console.print()
+ console.print(f"[green]✓[/green] {subject} are already configured!")
+ console.print()
+ console.print(f"[dim]Configuration file:[/dim] [cyan]{config_path}[/cyan]")
+ console.print()
+
+ return Confirm.ask(f"[bold]Would you like to reconfigure {action_verb}?[/bold]", default=False)
+
+ return False
+
+
+def build_initialization_panel(needs_config: bool, needs_inference: bool, needs_telemetry: bool, reset: bool) -> Panel:
+ """Build the initialization confirmation panel.
+
+ Args:
+ needs_config: Whether config initialization is needed.
+ needs_inference: Whether inference setup is needed.
+ needs_telemetry: Whether telemetry setup is needed.
+ reset: Whether this is a reset operation.
+
+ Returns:
+ A Panel containing the initialization confirmation message.
+ """
+ # Build message based on what's being initialized
+ message_parts: list[str] = []
+ if reset:
+ if needs_config:
+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")
+ if needs_inference:
+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] inference backends")
+ if needs_telemetry:
+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] telemetry preferences")
+ else:
+ if needs_config:
+ message_parts.append("• Create required configuration files in [cyan].pipelex/[/cyan]")
+ if needs_inference:
+ message_parts.append("• Ask you to choose your inference backends")
+ if needs_telemetry:
+ message_parts.append("• Ask you to choose your telemetry preferences")
+
+ # Determine title based on what's being initialized
+ num_items = sum([needs_config, needs_inference, needs_telemetry])
+ if reset:
+ if num_items > 1:
+ title_text = "[bold yellow]Resetting Configuration[/bold yellow]"
+ elif needs_config:
+ title_text = "[bold yellow]Resetting Configuration Files[/bold yellow]"
+ elif needs_inference:
+ title_text = "[bold yellow]Resetting Inference Backends[/bold yellow]"
+ else:
+ title_text = "[bold yellow]Resetting Telemetry[/bold yellow]"
+ elif num_items > 1:
+ title_text = "[bold cyan]Pipelex Initialization[/bold cyan]"
+ elif needs_config:
+ title_text = "[bold cyan]Configuration Setup[/bold cyan]"
+ elif needs_inference:
+ title_text = "[bold cyan]Inference Backend Setup[/bold cyan]"
+ else:
+ title_text = "[bold cyan]Telemetry Setup[/bold cyan]"
+
+ message = "\n".join(message_parts)
+ border_color = "yellow" if reset else "cyan"
+
+ return Panel(
+ message,
+ title=title_text,
+ border_style=border_color,
+ padding=(1, 2),
+ )
diff --git a/pipelex/cli/commands/run_cmd.py b/pipelex/cli/commands/run_cmd.py
index 5696d1bd5..5c85e3e28 100644
--- a/pipelex/cli/commands/run_cmd.py
+++ b/pipelex/cli/commands/run_cmd.py
@@ -16,9 +16,9 @@ from pipelex.pipelex import Pipelex
from pipelex.pipeline.execute import execute_pipeline
from pipelex.system.runtime import IntegrationMode
from pipelex.system.telemetry.events import EventProperty
-from pipelex.system.telemetry.telemetry_manager import PACKAGE_VERSION
from pipelex.tools.misc.file_utils import get_incremental_file_path
from pipelex.tools.misc.json_utils import JsonTypeError, load_json_dict_from_path, save_as_json_to_path
+from pipelex.tools.misc.package_utils import get_package_version
COMMAND = "run"
@@ -195,6 +195,6 @@ def run_cmd(
with new_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=COMMAND)
asyncio.run(run_pipeline(pipe_code=pipe_code, bundle_path=bundle_path))
diff --git a/pipelex/cli/commands/show_cmd.py b/pipelex/cli/commands/show_cmd.py
index 34e84d937..931f50a85 100644
--- a/pipelex/cli/commands/show_cmd.py
+++ b/pipelex/cli/commands/show_cmd.py
@@ -18,7 +18,7 @@ from pipelex.pipelex import Pipelex
from pipelex.system.configuration.config_loader import config_manager
from pipelex.system.runtime import IntegrationMode
from pipelex.system.telemetry.events import EventName, EventProperty
-from pipelex.system.telemetry.telemetry_manager import PACKAGE_VERSION
+from pipelex.tools.misc.package_utils import get_package_version
if TYPE_CHECKING:
from pipelex.cogt.models.model_manager import ModelManager
@@ -184,7 +184,7 @@ def list_pipes_cmd() -> None:
with new_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPES}")
do_list_pipes()
@@ -204,7 +204,7 @@ def show_pipe_cmd(
with new_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPE}")
do_show_pipe(pipe_code=pipe_code)
@@ -230,7 +230,7 @@ def show_models_cmd(
Pipelex.make(integration_mode=IntegrationMode.CLI)
with new_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_MODELS}")
asyncio.run(
@@ -256,7 +256,7 @@ def show_backends_cmd(
Pipelex.make(integration_mode=IntegrationMode.CLI)
with new_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_BACKENDS}")
do_show_backends(show_all=show_all_backends)
diff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.py
index 4a4e97248..916cc087c 100644
--- a/pipelex/cli/commands/validate_cmd.py
+++ b/pipelex/cli/commands/validate_cmd.py
@@ -19,7 +19,7 @@ from pipelex.pipe_run.dry_run import dry_run_pipe, dry_run_pipes
from pipelex.pipelex import Pipelex
from pipelex.system.runtime import IntegrationMode
from pipelex.system.telemetry.events import EventName, EventProperty
-from pipelex.system.telemetry.telemetry_manager import PACKAGE_VERSION
+from pipelex.tools.misc.package_utils import get_package_version
if TYPE_CHECKING:
from pipelex.core.validation_errors import ValidationErrorDetailsProtocol
@@ -33,7 +33,7 @@ def do_validate_all_libraries_and_dry_run() -> None:
pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
with new_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} all")
pipelex_instance.validate_libraries()
@@ -167,7 +167,7 @@ def validate_cmd(
with new_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
if bundle_path:
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} bundle")
else:
diff --git a/pipelex/cogt/model_backends/backend.py b/pipelex/cogt/model_backends/backend.py
index 9f019fad6..8aca3a190 100644
--- a/pipelex/cogt/model_backends/backend.py
+++ b/pipelex/cogt/model_backends/backend.py
@@ -8,6 +8,7 @@ from pipelex.system.configuration.config_model import ConfigModel
class InferenceBackend(ConfigModel):
name: str
+ display_name: str | None = None
enabled: bool = True
endpoint: str | None = None
api_key: str | None = None
diff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.py
index b5a2c1d2c..3f1dc15e8 100644
--- a/pipelex/cogt/model_backends/backend_library.py
+++ b/pipelex/cogt/model_backends/backend_library.py
@@ -14,8 +14,11 @@ from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.backend_factory import InferenceBackendBlueprint, InferenceBackendFactory
from pipelex.cogt.model_backends.model_spec_factory import InferenceModelSpecBlueprint, InferenceModelSpecFactory
from pipelex.config import get_config
+from pipelex.system.configuration.config_model import ConfigModel
+from pipelex.system.environment import get_optional_env
from pipelex.system.runtime import runtime_manager
-from pipelex.tools.misc.dict_utils import apply_to_strings_recursive
+from pipelex.tools.misc.dict_utils import apply_to_strings_recursive, extract_vars_from_strings_recursive
+from pipelex.tools.misc.placeholder import value_is_placeholder
from pipelex.tools.misc.toml_utils import load_toml_from_path
from pipelex.tools.secrets.secrets_utils import UnknownVarPrefixError, VarFallbackPatternError, VarNotFoundError, substitute_vars
from pipelex.types import Self
@@ -26,6 +29,31 @@ if TYPE_CHECKING:
InferenceBackendLibraryRoot = dict[str, InferenceBackend]
+class BackendCredentialStatus(ConfigModel):
+ """Status of a single credential variable."""
+
+ var_name: str
+ is_set: bool
+ is_placeholder: bool # True if value exists but is a placeholder like "${VAR}"
+
+
+class BackendCredentialsReport(ConfigModel):
+ """Report of credential status for a backend."""
+
+ backend_name: str
+ required_vars: list[str]
+ missing_vars: list[str]
+ placeholder_vars: list[str]
+ all_credentials_valid: bool
+
+
+class CredentialsValidationReport(ConfigModel):
+ """Complete report of credentials validation across all backends."""
+
+ backend_reports: dict[str, BackendCredentialsReport]
+ all_backends_valid: bool
+
+
class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
root: InferenceBackendLibraryRoot = Field(default_factory=dict)
@@ -139,6 +167,78 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
)
self.root[backend_name] = backend
+ def check_backend_credentials(self, path: str, include_disabled: bool = False) -> CredentialsValidationReport:
+ """Check if required environment variables are set for enabled backends.
+
+ This method loads backend configurations and extracts variable placeholders
+ without performing actual substitution or loading model specs.
+
+ Args:
+ path: Path to the backend library TOML file
+ include_disabled: If True, check disabled backends too
+
+ Returns:
+ CredentialsValidationReport with detailed status per backend
+
+ """
+ try:
+ backends_dict = load_toml_from_path(path=path)
+ except FileNotFoundError as file_not_found_exc:
+ msg = f"Could not find inference backend library at '{path}': {file_not_found_exc}"
+ raise InferenceBackendLibraryNotFoundError(msg) from file_not_found_exc
+
+ backend_reports: dict[str, BackendCredentialsReport] = {}
+ all_backends_valid = True
+
+ for backend_name, backend_dict in backends_dict.items():
+ enabled = backend_dict.get("enabled", True)
+ if not enabled and not include_disabled:
+ continue
+
+ # Skip internal backend
+ if backend_name == "internal":
+ continue
+
+ # Skip vertexai in CI testing
+ if runtime_manager.is_ci_testing and backend_name == "vertexai":
+ continue
+
+ # Extract all variable placeholders from the backend config
+ required_vars_set = extract_vars_from_strings_recursive(backend_dict)
+ required_vars = sorted(required_vars_set)
+
+ # Check status of each variable
+ missing_vars: list[str] = []
+ placeholder_vars: list[str] = []
+
+ for var_name in required_vars:
+ var_value = get_optional_env(var_name)
+ if var_value is None:
+ missing_vars.append(var_name)
+ elif value_is_placeholder(var_value):
+ placeholder_vars.append(var_name)
+
+ # Determine if all credentials are valid for this backend
+ backend_valid = len(missing_vars) == 0 and len(placeholder_vars) == 0
+
+ # Create report for this backend
+ backend_report = BackendCredentialsReport(
+ backend_name=backend_name,
+ required_vars=required_vars,
+ missing_vars=missing_vars,
+ placeholder_vars=placeholder_vars,
+ all_credentials_valid=backend_valid,
+ )
+ backend_reports[backend_name] = backend_report
+
+ if not backend_valid:
+ all_backends_valid = False
+
+ return CredentialsValidationReport(
+ backend_reports=backend_reports,
+ all_backends_valid=all_backends_valid,
+ )
+
def list_backend_names(self) -> list[str]:
return list(self.root.keys())
diff --git a/pipelex/kit/configs/inference/backends.toml b/pipelex/kit/configs/inference/backends.toml
index b4fbef7ce..62e51ab9a 100644
--- a/pipelex/kit/configs/inference/backends.toml
+++ b/pipelex/kit/configs/inference/backends.toml
@@ -1,41 +1,50 @@
[pipelex_inference]
+display_name = "⭐ Pipelex Inference"
+enabled = true
endpoint = "https://inference.pipelex.com/v1"
api_key = "${PIPELEX_INFERENCE_API_KEY}"
[azure_openai]
-enabled = true
+display_name = "Azure OpenAI"
+enabled = false
endpoint = "${AZURE_API_BASE}"
api_key = "${AZURE_API_KEY}"
api_version = "${AZURE_API_VERSION}"
[bedrock]
+display_name = "AWS Bedrock"
enabled = false
aws_region = "${AWS_REGION}"
[google]
+display_name = "Google AI"
enabled = false
api_key = "${GOOGLE_API_KEY}"
[vertexai]
+display_name = "Google Vertex AI"
enabled = false
gcp_project_id = "${GCP_PROJECT_ID}"
gcp_location = "${GCP_LOCATION}"
gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
[openai]
-enabled = true
+display_name = "OpenAI"
+enabled = false
api_key = "${OPENAI_API_KEY}"
[anthropic]
-enabled = true
+enabled = false
api_key = "${ANTHROPIC_API_KEY}"
claude_4_tokens_limit = 8192
[mistral]
-enabled = true
+display_name = "Mistral AI"
+enabled = false
api_key = "${MISTRAL_API_KEY}"
[xai]
+display_name = "xAI"
enabled = false
endpoint = "https://api.x.ai/v1"
api_key = "${XAI_API_KEY}"
@@ -45,11 +54,13 @@ enabled = false
endpoint = "http://localhost:11434/v1"
[blackboxai]
+display_name = "BlackBox AI"
enabled = false
endpoint = "https://api.blackbox.ai/v1"
api_key = "${BLACKBOX_API_KEY}"
[fal]
+display_name = "FAL"
enabled = false
api_key = "${FAL_API_KEY}"
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 978225131..13da1eebb 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -1,5 +1,4 @@
import os
-from importlib.metadata import metadata
from typing import Any, cast
from kajson.class_registry import ClassRegistry
@@ -58,6 +57,7 @@ from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME
from pipelex.system.telemetry.telemetry_manager import DO_NOT_TRACK_ENV_VAR_KEY, TelemetryManager
from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract, TelemetryManagerNoOp
from pipelex.test_extras.registry_test_models import TestRegistryModels
+from pipelex.tools.misc.package_utils import get_package_info
from pipelex.tools.misc.toml_utils import load_toml_from_path
from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
@@ -65,8 +65,7 @@ from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstr
from pipelex.types import Self
from pipelex.urls import URLs
-PACKAGE_NAME = __name__.split(".", maxsplit=1)[0]
-PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]
+PACKAGE_NAME, PACKAGE_VERSION = get_package_info()
class Pipelex(metaclass=MetaSingleton):
@@ -265,7 +264,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
self.telemetry_manager = telemetry_manager or TelemetryManager(telemetry_config=telemetry_config)
else:
self.telemetry_manager = TelemetryManagerNoOp()
- log.debug(f"Telemetry is disabled because the integration mode '{integration_mode}' does not allow it")
+ log.verbose(f"Telemetry is disabled because the integration mode '{integration_mode}' does not allow it")
self.telemetry_manager.setup(integration_mode=integration_mode)
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index c09065e1c..b24454d64 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -300,3 +300,6 @@ ocr = "model"
llm_handle = "model"
llm = "model"
llm_to_structure = "model_to_structure"
+
+
+[migration.migration_maps.telemetry]
diff --git a/pipelex/system/telemetry/telemetry_manager.py b/pipelex/system/telemetry/telemetry_manager.py
index 515fb678a..59c83a5eb 100644
--- a/pipelex/system/telemetry/telemetry_manager.py
+++ b/pipelex/system/telemetry/telemetry_manager.py
@@ -1,4 +1,3 @@
-from importlib.metadata import metadata
from typing import Any, Callable
import posthog
@@ -12,10 +11,9 @@ from pipelex.system.telemetry.events import EventName, EventProperty, Setting
from pipelex.system.telemetry.telemetry_config import TelemetryConfig, TelemetryMode
from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract
from pipelex.tools.log.log import log
+from pipelex.tools.misc.package_utils import get_package_version
DO_NOT_TRACK_ENV_VAR_KEY = "DO_NOT_TRACK"
-PACKAGE_NAME = __name__.split(".", maxsplit=1)[0]
-PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]
class TelemetryManager(TelemetryManagerAbstract):
@@ -89,15 +87,16 @@ class TelemetryManager(TelemetryManagerAbstract):
@override
def setup(self, integration_mode: IntegrationMode):
if telemetry_mode := TelemetryManagerAbstract.telemetry_was_just_enabled():
+ package_version = get_package_version()
with new_context():
tag(name=EventProperty.INTEGRATION, value=integration_mode)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.PIPELEX_VERSION, value=package_version)
tag(name=EventProperty.SETTING, value=Setting.TELEMETRY_MODE)
self.posthog.capture(
EventName.TELEMETRY_JUST_ENABLED,
properties={
EventProperty.TELEMETRY_MODE: telemetry_mode,
- EventProperty.PIPELEX_VERSION: PACKAGE_VERSION,
+ EventProperty.PIPELEX_VERSION: package_version,
},
)
diff --git a/pipelex/tools/misc/dict_utils.py b/pipelex/tools/misc/dict_utils.py
index 8db4d088f..27e7ff739 100644
--- a/pipelex/tools/misc/dict_utils.py
+++ b/pipelex/tools/misc/dict_utils.py
@@ -1,5 +1,6 @@
"""Dictionary utility functions for manipulating dictionary order and structure."""
+import re
from collections.abc import Callable
from typing import Any, TypeVar, cast
@@ -157,3 +158,68 @@ def substitute_nested_in_context(context: dict[str, Any], extra_params: dict[str
current[last_segment] = value
return context
+
+
+def extract_vars_from_strings_recursive(data: Any) -> set[str]:
+ """Recursively traverse a data structure and extract all variable placeholders.
+
+ This function walks through dictionaries, lists, and other nested structures,
+ extracting variable names from placeholders in the format ${VAR_NAME},
+ ${env:VAR}, ${secret:VAR}, and ${env:VAR|secret:VAR}.
+
+ Args:
+ data: The data structure to traverse (dict, list, or any value)
+
+ Returns:
+ Set of variable names found (without prefixes or ${} wrappers)
+
+ Example:
+ >>> data = {'a': 'hello ${USER}', 'b': {'c': '${env:HOME|secret:HOME_SECRET}'}}
+ >>> extract_vars_from_strings_recursive(data)
+ {'USER', 'HOME', 'HOME_SECRET'}
+
+ """
+ var_names: set[str] = set()
+
+ def extract_from_string(text: str) -> None:
+ """Extract variable names from a single string."""
+ # Pattern matches ${VAR_NAME} or ${prefix:VAR_NAME} or ${env:VAR|secret:VAR}
+ # Same pattern as in substitute_vars
+ pattern = r"\$\{([^}\n\"'$]+)\}"
+ matches = re.findall(pattern, text)
+
+ for var_spec in matches:
+ # Handle fallback pattern (contains |)
+ if "|" in var_spec:
+ parts = [part.strip() for part in var_spec.split("|")]
+ for part in parts:
+ if ":" in part:
+ # Extract variable name after prefix
+ _, var_name = part.split(":", 1)
+ var_names.add(var_name.strip())
+ else:
+ # No prefix
+ var_names.add(part)
+ # Handle prefixed variable (contains :)
+ elif ":" in var_spec:
+ _, var_name = var_spec.split(":", 1)
+ var_names.add(var_name.strip())
+ else:
+ # Simple variable without prefix
+ var_names.add(var_spec.strip())
+
+ def traverse(value: Any) -> None:
+ """Recursively traverse the data structure."""
+ if isinstance(value, dict):
+ dict_value: dict[Any, Any] = cast("dict[Any, Any]", value)
+ for v in dict_value.values():
+ traverse(v)
+ elif isinstance(value, list):
+ list_value: list[Any] = cast("list[Any]", value)
+ for item in list_value:
+ traverse(item)
+ elif isinstance(value, str):
+ extract_from_string(value)
+
+ traverse(data)
+ return var_names
diff --git a/pipelex/tools/misc/package_utils.py b/pipelex/tools/misc/package_utils.py
new file mode 100644
index 000000000..9331abac5
--- /dev/null
+++ b/pipelex/tools/misc/package_utils.py
@@ -0,0 +1,31 @@
+from importlib.metadata import metadata
+
+
+def get_package_name() -> str:
+ """Get the package name.
+
+ Returns:
+ str: The package name.
+ """
+ return __name__.split(".", maxsplit=1)[0]
+
+
+def get_package_info() -> tuple[str, str]:
+ """Get the package name and version.
+
+ Returns:
+ tuple[str, str]: A tuple of (package_name, package_version).
+ """
+ package_name = get_package_name()
+ package_version = metadata(package_name)["Version"]
+ return package_name, package_version
+
+
+def get_package_version() -> str:
+ """Get the package version.
+
+ Returns:
+ str: The package version.
+ """
+ _, package_version = get_package_info()
+ return package_version
diff --git a/pyproject.toml b/pyproject.toml
index e780319da..6b0af50f6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -165,7 +165,6 @@ reportPropertyTypeMismatch = true
reportRedeclaration = "error"
reportReturnType = "error"
reportSelfClsParameterName = "error"
-reportShadowedImports = true
reportTypeCommentUsage = "error"
reportTypedDictNotRequiredAccess = "error"
reportUnboundVariable = "error"
diff --git a/tests/integration/pipelex/cli/test_init_cmd.py b/tests/integration/pipelex/cli/test_init_cmd.py
new file mode 100644
index 000000000..67433563c
--- /dev/null
+++ b/tests/integration/pipelex/cli/test_init_cmd.py
@@ -0,0 +1,187 @@
+"""Integration tests for init command backend customization."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+from pipelex.cli.commands.init_cmd import customize_backends_config, init_config
+from pipelex.kit.paths import get_configs_dir
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit
+
+
+class TestBackendCustomization:
+ """Integration tests for backend customization during initialization."""
+
+ def test_customize_backends_config_with_default_selection(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test backend customization with default selection (pipelex_inference)."""
+ # Setup directories with actual backends.toml
+ inference_dir = tmp_path / ".pipelex" / "inference"
+ inference_dir.mkdir(parents=True)
+
+ # Copy actual backends.toml from kit
+ actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ test_backends = inference_dir / "backends.toml"
+ shutil.copy2(actual_backends, test_backends)
+
+ # Mock config_manager and user input
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+
+ # Mock Console and Prompt to simulate user selecting default [1]
+ mocker.patch("pipelex.cli.commands.init_cmd.Console")
+ mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="1")
+
+ # Execute
+ customize_backends_config()
+
+ # Verify backends.toml was customized
+ toml_doc = load_toml_with_tomlkit(str(test_backends))
+
+ # pipelex_inference should be enabled
+ assert "enabled" in toml_doc["pipelex_inference"] # type: ignore[operator]
+ assert toml_doc["pipelex_inference"]["enabled"] is True # type: ignore[index]
+
+ # Other backends should be disabled
+ for backend in ["openai", "anthropic", "mistral", "fal"]:
+ if backend in toml_doc and "enabled" in toml_doc[backend]: # type: ignore[operator]
+ assert toml_doc[backend]["enabled"] is False # type: ignore[index]
+
+ # internal backend should remain enabled
+ assert toml_doc["internal"]["enabled"] is True # type: ignore[index]
+
+ def test_customize_backends_config_with_multiple_selections(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test backend customization with multiple backend selections."""
+ # Setup directories
+ inference_dir = tmp_path / ".pipelex" / "inference"
+ inference_dir.mkdir(parents=True)
+
+ actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ test_backends = inference_dir / "backends.toml"
+ shutil.copy2(actual_backends, test_backends)
+
+ # Mock config_manager
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+
+ # Mock user input: select [6,7,8] (openai, anthropic, mistral) in 1-based indexing
+ mocker.patch("pipelex.cli.commands.init_cmd.Console")
+ mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="6,7,8")
+
+ # Execute
+ customize_backends_config()
+
+ # Verify customization
+ toml_doc = load_toml_with_tomlkit(str(test_backends))
+
+ # Selected backends should be enabled
+ assert toml_doc["openai"]["enabled"] is True # type: ignore[index]
+ assert toml_doc["anthropic"]["enabled"] is True # type: ignore[index]
+ assert toml_doc["mistral"]["enabled"] is True # type: ignore[index]
+
+ # pipelex_inference should be disabled
+ assert toml_doc["pipelex_inference"]["enabled"] is False # type: ignore[index]
+
+ # fal should be disabled
+ assert toml_doc["fal"]["enabled"] is False # type: ignore[index]
+
+ # internal should remain enabled
+ assert toml_doc["internal"]["enabled"] is True # type: ignore[index]
+
+ def test_customize_backends_config_with_space_separated_input(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test backend customization with space-separated input."""
+ inference_dir = tmp_path / ".pipelex" / "inference"
+ inference_dir.mkdir(parents=True)
+
+ actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ test_backends = inference_dir / "backends.toml"
+ shutil.copy2(actual_backends, test_backends)
+
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+
+ # Mock user input with spaces: "1 6 12" in 1-based indexing
+ mocker.patch("pipelex.cli.commands.init_cmd.Console")
+ mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="1 6 12")
+
+ # Execute
+ customize_backends_config()
+
+ # Verify customization
+ toml_doc = load_toml_with_tomlkit(str(test_backends))
+
+ # Selected backends should be enabled
+ assert toml_doc["pipelex_inference"]["enabled"] is True # type: ignore[index]
+ assert toml_doc["openai"]["enabled"] is True # type: ignore[index]
+ assert toml_doc["fal"]["enabled"] is True # type: ignore[index]
+
+ # Others should be disabled
+ assert toml_doc["anthropic"]["enabled"] is False # type: ignore[index]
+ assert toml_doc["mistral"]["enabled"] is False # type: ignore[index]
+
+ def test_init_config_copies_files_without_customizing(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test that init_config copies files but doesn't customize backends (that's init_cmd's job)."""
+ # Setup template directories
+ kit_configs_dir = tmp_path / "kit" / "configs"
+ kit_configs_dir.mkdir(parents=True)
+ (kit_configs_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '1.0'")
+
+ inference_dir = kit_configs_dir / "inference"
+ inference_dir.mkdir()
+
+ actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ shutil.copy2(actual_backends, inference_dir / "backends.toml")
+
+ # Setup target directory
+ target_dir = tmp_path / ".pipelex"
+ target_dir.mkdir()
+
+ # Mock config_manager
+ mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=str(kit_configs_dir))
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(target_dir)
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("typer.echo")
+
+ # Execute init_config
+ result = init_config(reset=False)
+
+ # Verify files were copied
+ assert result > 0
+ assert (target_dir / "pipelex.toml").exists()
+ assert (target_dir / "inference" / "backends.toml").exists()
+
+ # Verify backend customization was NOT applied (original values preserved)
+ toml_doc = load_toml_with_tomlkit(str(target_dir / "inference" / "backends.toml"))
+
+ # Verify original enabled states from template (template has most backends disabled)
+ assert toml_doc["pipelex_inference"]["enabled"] is True # type: ignore[index]
+ assert toml_doc["openai"]["enabled"] is False # type: ignore[index]
+ assert toml_doc["anthropic"]["enabled"] is False # type: ignore[index]
+ assert toml_doc["mistral"]["enabled"] is False # type: ignore[index]
+ assert toml_doc["internal"]["enabled"] is True # type: ignore[index]
+
+ def test_customize_backends_handles_missing_file_gracefully(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test that customize_backends_config handles missing backends.toml gracefully."""
+ # Setup directory WITHOUT backends.toml
+ config_dir = tmp_path / ".pipelex"
+ config_dir.mkdir()
+
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(config_dir)
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+
+ mock_console = mocker.patch("pipelex.cli.commands.init_cmd.Console")
+
+ # Execute - should not raise exception
+ customize_backends_config()
+
+ # Verify warning was printed
+ mock_console.return_value.print.assert_called()
diff --git a/tests/unit/cli/commands/test_init_cmd.py b/tests/unit/cli/test_init_cmd.py
similarity index 68%
rename from tests/unit/cli/commands/test_init_cmd.py
rename to tests/unit/cli/test_init_cmd.py
index 4c09966db..513a30594 100644
--- a/tests/unit/cli/commands/test_init_cmd.py
+++ b/tests/unit/cli/test_init_cmd.py
@@ -192,3 +192,85 @@ class TestInitCmd:
# Verify success message
calls = [call.args[0] for call in mock_echo.call_args_list]
assert any("✅ Copied 1 files to" in call for call in calls)
+
+ def test_do_init_config_dry_run_does_not_copy_files(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ # Setup template directories
+ kit_configs_dir = tmp_path / "kit" / "configs"
+ kit_configs_dir.mkdir(parents=True)
+ (kit_configs_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '1.0'")
+
+ # Create inference directory structure
+ inference_dir = kit_configs_dir / "inference"
+ inference_dir.mkdir()
+ (inference_dir / "backends.toml").write_text("[backends]\nconfig = 'value'")
+
+ # Setup target directory (empty initially)
+ target_dir = tmp_path / ".pipelex"
+ target_dir.mkdir()
+
+ # Mock config_manager
+ mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(target_dir)
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+
+ # Execute dry-run
+ count_dry = init_config(reset=False, dry_run=True)
+
+ # Verify count is correct but files were NOT actually copied
+ assert count_dry == 2
+ assert not (target_dir / "pipelex.toml").exists()
+ assert not (target_dir / "inference").exists()
+
+ def test_do_init_config_does_not_call_customize_backends(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ # Setup directories with inference structure
+ kit_configs_dir = tmp_path / "kit" / "configs"
+ kit_configs_dir.mkdir(parents=True)
+ (kit_configs_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '1.0'")
+
+ inference_dir = kit_configs_dir / "inference"
+ inference_dir.mkdir()
+ (inference_dir / "backends.toml").write_text("[openai]\nenabled = true\n[internal]\nenabled = true")
+
+ target_dir = tmp_path / ".pipelex"
+ target_dir.mkdir()
+
+ # Mock config_manager and customize function
+ mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(target_dir)
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+
+ mock_customize = mocker.patch("pipelex.cli.commands.init_cmd.customize_backends_config")
+ mocker.patch("typer.echo")
+
+ # Execute
+ init_config(reset=False)
+
+ # Verify customize_backends_config was NOT called (separation of concerns)
+ mock_customize.assert_not_called()
+
+ def test_do_init_config_skips_customize_when_no_files_copied(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ # Setup directories with all files already existing
+ kit_configs_dir = tmp_path / "kit" / "configs"
+ kit_configs_dir.mkdir(parents=True)
+ (kit_configs_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '1.0'")
+
+ target_dir = tmp_path / ".pipelex"
+ target_dir.mkdir()
+ (target_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '0.9'")
+
+ # Mock config_manager and customize function
+ mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mock_config_manager = mocker.MagicMock()
+ mock_config_manager.pipelex_config_dir = str(target_dir)
+ mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+
+ mock_customize = mocker.patch("pipelex.cli.commands.init_cmd.customize_backends_config")
+ mocker.patch("typer.echo")
+
+ # Execute
+ init_config(reset=False)
+
+ # Verify customize_backends_config was NOT called (no files copied)
+ mock_customize.assert_not_called()