Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/merge model fallback (#431)

Louis Choquel918f52f646ed
diff --git a/.badges/tests.json b/.badges/tests.json
index 3c153507c..2440a5710 100644
--- a/.badges/tests.json
+++ b/.badges/tests.json
@@ -1,7 +1,7 @@
{
"schemaVersion": 1,
"label": "tests",
- "message": "1384",
+ "message": "1434",
"color": "blue",
"cacheSeconds": 300
}
\ No newline at end of file
diff --git a/.blackboxrules b/.blackboxrules
index 23c6f20aa..6e236d028 100644
--- a/.blackboxrules
+++ b/.blackboxrules
@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/.cursor/rules/llms.mdc b/.cursor/rules/llms.mdc
index 8babc7660..7622b337d 100644
--- a/.cursor/rules/llms.mdc
+++ b/.cursor/rules/llms.mdc
@@ -55,7 +55,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/.cursor/rules/write_pipelex.mdc b/.cursor/rules/write_pipelex.mdc
index c64d617ee..b46e4f23d 100644
--- a/.cursor/rules/write_pipelex.mdc
+++ b/.cursor/rules/write_pipelex.mdc
@@ -824,7 +824,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 23c6f20aa..6e236d028 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/.pipelex/inference/backends/anthropic.toml b/.pipelex/inference/backends/anthropic.toml
index 4b57964c4..66721e2c1 100644
--- a/.pipelex/inference/backends/anthropic.toml
+++ b/.pipelex/inference/backends/anthropic.toml
@@ -89,3 +89,11 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+
+["claude-4.5-haiku"]
+model_id = "claude-haiku-4-5-20251001"
+max_tokens = 64000
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 100
+costs = { input = 1.0, output = 5.0 }
diff --git a/.pipelex/inference/backends/azure_openai.toml b/.pipelex/inference/backends/azure_openai.toml
index 135638ad5..930f161fd 100644
--- a/.pipelex/inference/backends/azure_openai.toml
+++ b/.pipelex/inference/backends/azure_openai.toml
@@ -80,6 +80,12 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.1, output = 4.4 }
+[o3]
+model_id = "o3-2025-04-16"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 2, output = 8 }
+
# --- GPT-5 Series -------------------------------------------------------------
[gpt-5-mini]
model_id = "gpt-5-mini-2025-08-07"
diff --git a/.pipelex/inference/backends/bedrock.toml b/.pipelex/inference/backends/bedrock.toml
index c27baf0b5..cea91344d 100644
--- a/.pipelex/inference/backends/bedrock.toml
+++ b/.pipelex/inference/backends/bedrock.toml
@@ -92,8 +92,6 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
-# anthropic.claude-sonnet-4-5-20250929-v1:0,Anthropic,arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0,
-
["claude-4.5-sonnet"]
sdk = "bedrock_anthropic"
model_id = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
@@ -102,3 +100,12 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+
+["claude-4.5-haiku"]
+sdk = "bedrock_anthropic"
+model_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
+max_tokens = 8192
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 100
+costs = { input = 1.0, output = 5.0 }
diff --git a/.pipelex/inference/backends/pipelex_inference.toml b/.pipelex/inference/backends/pipelex_inference.toml
index 805b761ad..e516e274e 100644
--- a/.pipelex/inference/backends/pipelex_inference.toml
+++ b/.pipelex/inference/backends/pipelex_inference.toml
@@ -103,6 +103,12 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 3, output = 15 }
+["claude-4.5-haiku"]
+model_id = "pipelex/claude-4.5-haiku"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+costs = { input = 1, output = 5 }
+
# --- Gemini LLMs --------------------------------------------------------------
["gemini-2.0-flash"]
model_id = "gemini/gemini-2.0-flash"
diff --git a/.pipelex/inference/deck/base_deck.toml b/.pipelex/inference/deck/base_deck.toml
index 9b2d06f13..12f516755 100644
--- a/.pipelex/inference/deck/base_deck.toml
+++ b/.pipelex/inference/deck/base_deck.toml
@@ -28,27 +28,59 @@ best-gemini = "gemini-2.5-pro"
best-mistral = "mistral-medium"
best-grok = "grok-3"
-cheap-gpt = "gpt-4o-mini"
-cheap_llm_for_text = "gpt-4o-mini"
-cheap_llm_for_object = "gpt-4o-mini"
-cheap_llm_for_vision = "gemini-2.5-flash-lite"
+# Image generation aliases
+base-img-gen = "flux-pro/v1.1"
+best-img-gen = "flux-pro/v1.1-ultra"
+fast-img-gen = "fast-lightning-sdxl"
-base_llm_for_text = "gpt-4o-mini"
-base_llm_for_object = "gpt-4o-mini"
+####################################################################################################
+# Waterfalls
+####################################################################################################
+[waterfalls]
smart_llm = [
"claude-4.5-sonnet",
"claude-4.1-opus",
"claude-4-sonnet",
"gemini-2.5-pro",
- "gpt-5",
+ "gpt-4o", # we're using gpt-4o here rather than gpt-5 until we'll handle reasoning parameters
+ "grok-3",
+]
+smart_llm_with_vision = [
+ "claude-4.5-sonnet",
+ "claude-4.1-opus",
+ "claude-4-sonnet",
+ "gemini-2.5-pro",
+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters
+ "grok-3",
+]
+smart_llm_for_structured = [
+ "claude-4.5-sonnet",
+ "claude-4.1-opus",
+ "claude-4-sonnet",
+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters
]
-llm_for_large_codebase = ["gemini-2.5-pro", "claude-4.5-sonnet", "gpt-5"]
-# Image generation aliases
-base-img-gen = "flux-pro/v1.1-ultra"
-best-img-gen = "nano-banana"
-fast-img-gen = "fast-lightning-sdxl"
+llm_for_large_codebase = [
+ "gemini-2.5-pro",
+ "claude-4.5-sonnet",
+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters
+ "gemini-2.5-flash",
+ "grok-3",
+]
+cheap_llm = [
+ "gpt-4o-mini",
+ "gemini-2.5-flash-lite",
+ "mistral-small",
+ "claude-3-haiku",
+ "grok-3-mini",
+]
+cheap_llm_for_vision = [
+ "gpt-4o-mini",
+ "gemini-2.5-flash-lite",
+ "claude-3-haiku",
+]
+cheap_llm_for_structured = ["gpt-4o-mini", "mistral-small", "claude-3-haiku"]
####################################################################################################
# LLM Presets
@@ -56,73 +88,47 @@ fast-img-gen = "fast-lightning-sdxl"
[llm.presets]
-####################################################################################################
-# LLM Presets — General purpose
-
-cheap_llm_for_text = { model = "cheap_llm_for_text", temperature = 0.5 }
-cheap_llm_for_short_text = { model = "cheap_llm_for_text", temperature = 0.5, max_tokens = 50 }
-cheap_llm_for_object = { model = "cheap_llm_for_object", temperature = 0.5 }
-cheap_llm_to_structure = { model = "cheap_llm_for_object", temperature = 0.1 }
-
-llm_for_testing_gen_text = { model = "base_llm_for_text", temperature = 0.5 }
-llm_for_testing_gen_object = { model = "base_llm_for_object", temperature = 0.5 }
-llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }
-llm_for_testing_vision_structured = { model = "cheap-gpt", temperature = 0.5 }
-
-####################################################################################################
-# LLM Presets — Specific skills
-
-# Cheap llm
-llm_cheap = { model = "base-gemini", temperature = 0.1 }
+# LLM Presets — Specific skills -------------------------------------------------------------
# Generation skills
llm_for_factual_writing = { model = "base-gpt", temperature = 0.1 }
llm_for_creative_writing = { model = "base-gpt", temperature = 0.9 }
-llm_to_write_questions = { model = "gpt-4o-mini", temperature = 0.3 }
-
-# Reasoning skills
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
-llm_to_reason_on_diagram = { model = "base-claude", temperature = 0.5 }
-llm_to_analyze_data = { model = "base-claude", temperature = 0.5 }
+llm_for_writing_cheap = { model = "gpt-4o-mini", temperature = 0.3 }
-# Search and answer skills
-llm_to_answer_easy_questions = { model = "gpt-4o-mini", temperature = 0.3 }
-llm_to_answer_hard_questions = { model = "base-claude", temperature = 0.3 }
+# Retrieve and answer questions skills
+llm_to_answer_questions_cheap = { model = "gpt-4o-mini", temperature = 0.3 }
+llm_to_answer_questions = { model = "base-claude", temperature = 0.3 }
llm_to_retrieve = { model = "base-claude", temperature = 0.1 }
-llm_for_enrichment = { model = "gpt-4o", temperature = 0.1 }
-llm_to_enrich = { model = "base-claude", temperature = 0.1 }
-llm_for_question_and_excerpt_reformulation = { model = "gpt-4o", temperature = 0.9 }
# Engineering skills
-llm_to_engineer = { model = "smart_llm", temperature = 0.2 }
-llm_to_write = { model = "base-gemini", temperature = 0.2 }
-llm_to_pipelex = { model = "base-claude", temperature = 0.2 }
+llm_to_engineer = { model = "smart_llm_for_structured", temperature = 0.2 }
llm_to_code = { model = "base-claude", temperature = 0.1 }
llm_to_analyze_large_codebase = { model = "base-claude", temperature = 0.1 }
-# Image skills
-llm_for_basic_vision = { model = "gpt-4o-mini", temperature = 0.1 }
-llm_to_write_img_gen_prompt = { model = "base-claude", temperature = 0.2 }
-llm_to_describe_img = { model = "base-claude", temperature = 0.5 }
-llm_for_visual_analysis = { model = "base-claude", temperature = 0.5 }
-llm_for_visual_design = { model = "base-claude", temperature = 0.5 }
-llm_to_design_fashion = { model = "base-claude", temperature = 0.7 }
+# Vision skills
+llm_for_img_to_text_cheap = { model = "gpt-4o-mini", temperature = 0.1 }
llm_for_img_to_text = { model = "base-claude", temperature = 0.1 }
+llm_for_diagram_to_text = { model = "base-claude", temperature = 0.3 }
+llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }
-# Extraction skills
-llm_to_extract_diagram = { model = "base-claude", temperature = 0.5 }
-llm_to_extract_invoice = { model = "claude-4.5-sonnet", temperature = 0.1 }
-llm_to_extract_invoice_from_scan = { model = "base-claude", temperature = 0.5 }
-llm_to_extract_legal_terms = { model = "base-claude", temperature = 0.1 }
-llm_to_extract_tables = { model = "base-claude", temperature = 0.1 }
+# Image generation prompting skills
+llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }
+llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }
+
+# LLM Presets — For Testing ---------------------------------------------------------------------
+
+llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }
+llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.5 }
+llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }
+llm_for_testing_vision_structured = { model = "cheap_llm_for_vision", temperature = 0.5 }
####################################################################################################
# LLM Choices
####################################################################################################
[llm.choice_defaults]
-for_text = "cheap_llm_for_text"
-for_object = "cheap_llm_for_object"
+for_text = "cheap_llm"
+for_object = "cheap_llm_for_structured"
####################################################################################################
# Extract Presets
@@ -144,7 +150,6 @@ choice_default = "gen_image_basic"
[img_gen.presets]
-
# General purpose
gen_image_basic = { model = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }
gen_image_fast = { model = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }
diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.toml
index 4a77d257d..592533daf 100644
--- a/.pipelex/inference/routing_profiles.toml
+++ b/.pipelex/inference/routing_profiles.toml
@@ -1,12 +1,12 @@
# Routing profile library - Routes models to their backends
-# ================================================
+# =========================================================================================
# This file controls which backend serves which model.
# Simply change the 'active' field to switch profiles,
# or you can add your own custom profiles.
#
# Documentation: https://docs.pipelex.com
# Support: https://go.pipelex.com/discord
-# ================================================
+# =========================================================================================
# Which profile to use (change this to switch routing)
active = "pipelex_first"
@@ -21,13 +21,21 @@ active = "pipelex_first"
# 3. The .pipelex/inference/backends.toml is already configured with api_key = "${PIPELEX_INFERENCE_API_KEY}"
# which will get the key from the environment variable.
-# ============================================
+# =========================================================================================
# Routing Profiles
-# ============================================
+# =========================================================================================
[profiles.pipelex_first]
description = "Use Pipelex Inference backend for all its supported models"
default = "pipelex_inference"
+fallback_order = [
+ "pipelex_inference",
+ "azure_openai",
+ "bedrock",
+ "blackboxai",
+ "mistral",
+ "fal",
+]
[profiles.pipelex_first.routes]
# Pattern matching: "model-pattern" = "backend-name"
@@ -35,37 +43,82 @@ default = "pipelex_inference"
"claude-*" = "pipelex_inference"
"grok-*" = "pipelex_inference"
"gemini-*" = "pipelex_inference"
+
+[profiles.pipelex_first.optional_routes] # Each optional route is considered only if its backend is available
"*-sdxl" = "blackboxai"
"flux-*" = "blackboxai"
"nano-banana" = "blackboxai"
"gpt-image-1" = "openai"
"mistral-ocr" = "mistral"
-[profiles.all_blackboxai]
-description = "Use BlackBoxAI backend for all its supported models"
-default = "blackboxai"
+[profiles.all_anthropic]
+description = "Use Anthropic backend for all its supported models"
+default = "anthropic"
+
+[profiles.all_azure_openai]
+description = "Use Azure OpenAI backend for all its supported models"
+default = "azure_openai"
[profiles.all_bedrock]
description = "Use Bedrock backend for all its supported models"
default = "bedrock"
-[profiles.all_anthropic]
-description = "Use Anthropic backend for all its supported models"
-default = "anthropic"
+[profiles.all_blackboxai]
+description = "Use BlackBoxAI backend for all its supported models"
+default = "blackboxai"
+
+[profiles.all_fal]
+description = "Use FAL backend for all its supported models"
+default = "fal"
-[profiles.all_gemini]
+[profiles.all_google]
description = "Use Google GenAI backend for all its supported models"
default = "google"
-[profiles.all_azure_openai]
-description = "Use Azure OpenAI backend for all its supported models"
-default = "azure_openai"
+[profiles.all_mistral]
+description = "Use Mistral backend for all its supported models"
+default = "mistral"
+
+[profiles.all_ollama]
+description = "Use Ollama backend for all its supported models"
+default = "ollama"
+
+[profiles.all_openai]
+description = "Use OpenAI backend for all its supported models"
+default = "openai"
+
+[profiles.all_vertexai]
+description = "Use Vertex AI backend for all its supported models"
+default = "vertexai"
+
+[profiles.all_xai]
+description = "Use xAI backend for all its supported models"
+default = "xai"
+
+# =========================================================================================
+# Custom Profiles
+# =========================================================================================
+# Add your own profiles below following the same pattern:
+#
+# [profiles.your_profile_name]
+# description = "What this profile does"
+# default = "backend-name" # Where to route models by default
+# [profiles.your_profile_name.routes]
+# "model-pattern" = "backend-name" # Specific routing rules
+#
+# Pattern matching supports:
+# - Exact names: "gpt-4o-mini"
+# - Wildcards: "claude-*" (matches all models starting with claude-)
+# - Partial wildcards: "*-sonnet" (matches all sonnet variants)
-[profiles.custom_routing]
-description = "Custom routing"
+# =========================================================================================
+# Example of a custom routing profile with mostly pattern matching and one specific model
+# =========================================================================================
+[profiles.example_routing_using_patterns]
+description = "Example routing profile using patterns"
default = "pipelex_inference"
-[profiles.custom_routing.routes]
+[profiles.example_routing_using_patterns.routes]
# Pattern matching: "model-pattern" = "backend-name"
"gpt-*" = "azure_openai"
"claude-*" = "bedrock"
@@ -75,10 +128,14 @@ default = "pipelex_inference"
"flux-*" = "fal"
"gpt-image-1" = "openai"
-[profiles.to_test_various_backends]
-description = "Route models to various backends"
+# =========================================================================================
+# Example of a custom routing profile with specific model matching
+# =========================================================================================
+
+[profiles.example_routing_using_specific_models]
+description = "Example routing profile using specific models"
-[profiles.to_test_various_backends.routes]
+[profiles.example_routing_using_specific_models.routes]
"gpt-5-nano" = "pipelex_inference"
"gpt-4o-mini" = "blackboxai"
"gpt-5-mini" = "openai"
@@ -93,19 +150,3 @@ description = "Route models to various backends"
"grok-3" = "pipelex_inference"
"grok-3-mini" = "xai"
-
-# ============================================
-# Custom Profiles
-# ============================================
-# Add your own profiles below following the same pattern:
-#
-# [profiles.your_profile_name]
-# description = "What this profile does"
-# default = "backend-name" # Where to route models by default
-# [profiles.your_profile_name.routes]
-# "model-pattern" = "backend-name" # Specific routing rules
-#
-# Pattern matching supports:
-# - Exact names: "gpt-4o-mini"
-# - Wildcards: "claude-*" (matches all models starting with claude-)
-# - Partial wildcards: "*-sonnet" (matches all sonnet variants)
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 1827cff67..def33534c 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -47,9 +47,7 @@
"program": "${workspaceFolder}/.venv/bin/pipelex",
"args": [
"validate",
- "describe_image",
- "-c",
- "pipelex/libraries",
+ "temp/bundle.plx",
],
"console": "integratedTerminal",
"justMyCode": false
diff --git a/.windsurfrules.md b/.windsurfrules.md
index 23c6f20aa..6e236d028 100644
--- a/.windsurfrules.md
+++ b/.windsurfrules.md
@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/AGENTS.md b/AGENTS.md
index 23c6f20aa..6e236d028 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4a6688e78..e57c5b0e3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
# Changelog
+## Unreleased
+
+- Updated the api documentation.
+- Updated the routing profiles.
+
## [v0.14.3] - 2025-10-29
### Added
diff --git a/CLAUDE.md b/CLAUDE.md
index 23c6f20aa..6e236d028 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/Makefile b/Makefile
index 348132ee4..6b104d414 100644
--- a/Makefile
+++ b/Makefile
@@ -458,7 +458,7 @@ check-TODOs: env
docs: env
$(call PRINT_TITLE,"Serving documentation with mkdocs")
- $(VENV_MKDOCS) serve -a 127.0.0.1:8000 -f $(CURDIR)/mkdocs.yml --watch $(CURDIR)/docs -s
+ $(VENV_MKDOCS) serve -a 127.0.0.1:8000 -f "$(CURDIR)/mkdocs.yml" --watch "$(CURDIR)/docs" -s
docs-check: env
$(call PRINT_TITLE,"Checking documentation build with mkdocs")
diff --git a/README.md b/README.md
index 074eaaabc..7dce15f0d 100644
--- a/README.md
+++ b/README.md
@@ -137,7 +137,7 @@ Performs comprehensive analysis comparing the candidate's CV against the job off
"""
inputs = { cv_pages = "Page[]", job_offer_pages = "Page[]" }
output = "MatchAnalysis"
-model = "llm_to_answer_hard_questions"
+model = "llm_to_answer_questions"
system_prompt = """
You are an expert HR analyst and recruiter specializing in candidate-job fit assessment. Your task is to generate a structured MatchAnalysis comparing a candidate's CV against job requirements.
"""
@@ -160,7 +160,7 @@ Generates exactly 5 targeted, relevant interview questions based on the match an
"""
inputs = { match_analysis = "MatchAnalysis" }
output = "Question[5]"
-model = "llm_to_write_questions"
+model = "llm_for_writing_cheap"
system_prompt = """
You are an expert HR interviewer and talent assessment specialist. Your task is to generate structured interview questions based on candidate-position match analysis.
"""
diff --git a/pipelex/builder/builder_errors.py b/pipelex/builder/builder_errors.py
index 86c1b8b64..79457e77d 100644
--- a/pipelex/builder/builder_errors.py
+++ b/pipelex/builder/builder_errors.py
@@ -20,7 +20,7 @@ from pipelex.builder.validation_error_data import (
StaticValidationErrorData,
)
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.validation_errors import ValidationErrorDetailsProtocol
+from pipelex.core.validation_errors import PipelexValidationExceptionAbstract
from pipelex.exceptions import PipelexException
from pipelex.types import Self
@@ -72,7 +72,7 @@ class ValidateDryRunError(Exception):
"""Raised when validating the dry run of a pipe."""
-class PipelexBundleError(PipelexException, ValidationErrorDetailsProtocol):
+class PipelexBundleError(PipelexException, PipelexValidationExceptionAbstract):
"""Main bundle error that aggregates multiple types of errors."""
def __init__(
diff --git a/pipelex/builder/pipe/pipe_llm_spec.py b/pipelex/builder/pipe/pipe_llm_spec.py
index 03bb5d864..4d2fce330 100644
--- a/pipelex/builder/pipe/pipe_llm_spec.py
+++ b/pipelex/builder/pipe/pipe_llm_spec.py
@@ -14,15 +14,14 @@ if TYPE_CHECKING:
class LLMSkill(StrEnum):
LLM_TO_RETRIEVE = "llm_to_retrieve"
- LLM_TO_ANSWER_EASY_QUESTIONS = "llm_to_answer_easy_questions"
- LLM_TO_ANSWER_HARD_QUESTIONS = "llm_to_answer_hard_questions"
- LLM_TO_WRITE_QUESTIONS = "llm_to_write_questions"
- LLM_FOR_BASIC_VISION = "llm_for_basic_vision"
- LLM_FOR_VISUAL_ANALYSIS = "llm_for_visual_analysis"
+
+ LLM_TO_ANSWER_QUESTIONS_CHEAP = "llm_to_answer_questions_cheap"
+ LLM_TO_ANSWER_QUESTIONS = "llm_to_answer_questions"
+
+ LLM_FOR_WRITING_CHEAP = "llm_for_writing_cheap"
+ LLM_FOR_IMG_TO_TEXT_CHEAP = "llm_for_img_to_text_cheap"
LLM_FOR_VISUAL_DESIGN = "llm_for_visual_design"
LLM_FOR_CREATIVE_WRITING = "llm_for_creative_writing"
- LLM_TO_REASON_ON_DIAGRAM = "llm_to_reason_on_diagram"
- LLM_TO_ANALYZE_DATA = "llm_to_analyze_data"
LLM_TO_CODE = "llm_to_code"
LLM_TO_ANALYZE_LARGE_CODEBASE = "llm_to_analyze_large_codebase"
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 6ce815e04..ed131e5c6 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -8,7 +8,8 @@ from typing_extensions import override
from pipelex.cli.commands.build_cmd import build_app
from pipelex.cli.commands.doctor_cmd import doctor_cmd
-from pipelex.cli.commands.init_cmd import InitFocus, init_cmd
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.ui.types import InitFocus
from pipelex.cli.commands.kit_cmd import kit_app
from pipelex.cli.commands.run_cmd import run_cmd
from pipelex.cli.commands.show_cmd import show_app
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index 09a1bd842..7c66ac744 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -4,15 +4,29 @@ from typing import TYPE_CHECKING, Annotated
import click
import typer
-from posthog import new_context, tag
+from posthog import tag
from pipelex import pretty_print
from pipelex.builder.builder import PipelexBundleSpec, load_and_validate_bundle
from pipelex.builder.builder_errors import PipeBuilderError, PipelexBundleError
from pipelex.builder.builder_loop import BuilderLoop
from pipelex.builder.runner_code import generate_input_memory_json_string, generate_runner_code
-from pipelex.exceptions import PipeInputError, PipelineExecutionError
-from pipelex.hub import get_report_delegate, get_required_pipe
+from pipelex.cli.error_handlers import (
+ ErrorContext,
+ handle_model_availability_error,
+ handle_model_choice_error,
+ handle_model_deck_preset_error,
+ handle_validation_error,
+)
+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
+from pipelex.exceptions import (
+ LibraryLoadingError,
+ PipeInputError,
+ PipelineExecutionError,
+ PipeOperatorModelAvailabilityError,
+ PipeOperatorModelChoiceError,
+)
+from pipelex.hub import get_report_delegate, get_required_pipe, get_telemetry_manager
from pipelex.language.plx_factory import PlxFactory
from pipelex.pipelex import PACKAGE_VERSION, Pipelex
from pipelex.pipeline.execute import execute_pipeline
@@ -82,7 +96,13 @@ def build_pipe_cmd(
typer.Option("--no-output", help="Skip saving the pipeline to file"),
] = False,
) -> None:
- Pipelex.make(integration_mode=IntegrationMode.CLI)
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PIPE)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PIPE)
+
typer.secho("🔥 Starting pipe builder... 🚀\n", fg=typer.colors.GREEN)
async def run_pipeline():
@@ -136,17 +156,27 @@ def build_pipe_cmd(
except Exception as exc:
typer.secho(f"⚠️ Warning: Could not generate input JSON: {exc}", fg=typer.colors.YELLOW)
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPE}")
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPE}")
+
+ start_time = time.time()
+ asyncio.run(run_pipeline())
+ end_time = time.time()
+ typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)
+
+ get_report_delegate().generate_report()
+
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.BUILD)
- start_time = time.time()
- asyncio.run(run_pipeline())
- end_time = time.time()
- typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.BUILD)
- get_report_delegate().generate_report()
+ finally:
+ pipelex_instance.teardown()
@build_app.command(SUB_COMMAND_RUNNER, help="Build the Python code to run a pipe with the necessary inputs")
@@ -229,9 +259,6 @@ def prepare_runner_cmd(
raise typer.Exit(1)
async def prepare_runner(pipe_code: str | None = None, bundle_path: str | None = None):
- # Initialize Pipelex
- Pipelex.make(integration_mode=IntegrationMode.CLI)
-
if bundle_path:
try:
bundle_blueprint = await load_and_validate_bundle(bundle_path)
@@ -287,12 +314,29 @@ def prepare_runner_cmd(
typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)
raise typer.Exit(1) from exc
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)
+
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")
+
+ asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))
- asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.BUILD)
+
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.BUILD)
+
+ finally:
+ pipelex_instance.teardown()
@build_app.command(SUB_COMMAND_INPUTS, help="Generate example input JSON for a pipe")
@@ -371,9 +415,6 @@ def generate_inputs_cmd(
raise typer.Exit(1)
async def generate_inputs(pipe_code: str | None = None, bundle_path: str | None = None):
- # Initialize Pipelex
- Pipelex.make(integration_mode=IntegrationMode.CLI)
-
if bundle_path:
try:
bundle_blueprint = await load_and_validate_bundle(bundle_path)
@@ -425,12 +466,29 @@ def generate_inputs_cmd(
typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)
raise typer.Exit(1) from exc
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)
- asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")
+
+ asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))
+
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.BUILD)
+
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.BUILD)
+
+ finally:
+ pipelex_instance.teardown()
@build_app.command(SUB_COMMAND_ONE_SHOT_PIPE, help="Developer utility for contributors: deliver pipeline in one shot, without validation loop")
@@ -452,7 +510,13 @@ def build_one_shot_cmd(
typer.Option("--no-output", help="Skip saving the pipeline to file"),
] = False,
) -> None:
- Pipelex.make(integration_mode=IntegrationMode.CLI)
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_ONE_SHOT)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_ONE_SHOT)
+
typer.secho("🔥 Starting pipe builder... 🚀\n", fg=typer.colors.GREEN)
async def run_pipeline():
@@ -484,17 +548,27 @@ def build_one_shot_cmd(
save_text_to_path(text=plx_content, path=output_path)
typer.secho(f"\n✅ Pipeline saved to: {output_path}", fg=typer.colors.GREEN)
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_ONE_SHOT_PIPE}")
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_ONE_SHOT_PIPE}")
+
+ start_time = time.time()
+ asyncio.run(run_pipeline())
+ end_time = time.time()
+ typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)
- start_time = time.time()
- asyncio.run(run_pipeline())
- end_time = time.time()
- typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)
+ get_report_delegate().generate_report()
- get_report_delegate().generate_report()
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.BUILD)
+
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.BUILD)
+
+ finally:
+ pipelex_instance.teardown()
@build_app.command(
@@ -527,7 +601,13 @@ def build_partial_cmd(
typer.Option("--no-output", help="Skip saving the pipeline to file"),
] = False,
) -> None:
- Pipelex.make(integration_mode=IntegrationMode.CLI)
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PARTIAL)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PARTIAL)
+
typer.secho("🔥 Starting pipe builder... 🚀\n", fg=typer.colors.GREEN)
async def run_pipeline():
@@ -577,14 +657,24 @@ def build_partial_cmd(
else:
typer.secho("\n⚠️ Pipeline not saved to file (--no-output specified)", fg=typer.colors.YELLOW)
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PARTIAL_PIPE}")
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PARTIAL_PIPE}")
+
+ start_time = time.time()
+ asyncio.run(run_pipeline())
+ end_time = time.time()
+ typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)
+
+ get_report_delegate().generate_report()
+
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.BUILD)
- start_time = time.time()
- asyncio.run(run_pipeline())
- end_time = time.time()
- typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.BUILD)
- get_report_delegate().generate_report()
+ finally:
+ pipelex_instance.teardown()
diff --git a/pipelex/cli/commands/doctor_cmd.py b/pipelex/cli/commands/doctor_cmd.py
index d961cd437..cc6daab43 100644
--- a/pipelex/cli/commands/doctor_cmd.py
+++ b/pipelex/cli/commands/doctor_cmd.py
@@ -12,10 +12,16 @@ from rich.panel import Panel
from rich.prompt import Confirm
from rich.text import Text
-from pipelex.cli.commands.init_cmd import InitFocus, init_cmd, init_config
+from pipelex import log
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.config_files import init_config
+from pipelex.cli.commands.init.ui.types import InitFocus
from pipelex.cogt.model_backends.backend_library import BackendCredentialsReport
-from pipelex.config import PipelexConfig
+from pipelex.cogt.models.model_manager import ModelManager
+from pipelex.config import PipelexConfig, get_config
from pipelex.core.validation import report_validation_error
+from pipelex.exceptions import PipelexConfigError
+from pipelex.hub import PipelexHub, set_pipelex_hub
from pipelex.system.configuration.config_loader import config_manager
from pipelex.system.environment import get_optional_env
from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryConfig
@@ -23,6 +29,7 @@ from pipelex.tools.misc.dict_utils import extract_vars_from_strings_recursive
from pipelex.tools.misc.file_utils import path_exists
from pipelex.tools.misc.placeholder import value_is_placeholder
from pipelex.tools.misc.toml_utils import load_toml_from_path
+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
@@ -166,6 +173,8 @@ def display_health_report(
backends_healthy: bool,
backends_message: str,
backend_reports: dict[str, BackendCredentialsReport],
+ models_healthy: bool,
+ models_message: str,
) -> None:
"""Display a comprehensive health report.
@@ -179,8 +188,10 @@ def display_health_report(
backends_healthy: Whether backends check passed
backends_message: Message about backends status
backend_reports: Dict of backend credential reports
+ models_healthy: Whether models check passed
+ models_message: Message about models status
"""
- all_healthy = config_healthy and telemetry_healthy and backends_healthy
+ all_healthy = config_healthy and telemetry_healthy and backends_healthy and models_healthy
# Overall status panel
if all_healthy:
@@ -238,6 +249,14 @@ def display_health_report(
console.print(f" [yellow]⚠[/yellow] Placeholders: {', '.join(backend_report.placeholder_vars)}")
console.print()
+ # Models section
+ console.print("[bold]Models[/bold]")
+ if models_healthy:
+ console.print(f" [green]✓[/green] {models_message}")
+ else:
+ console.print(f" [red]✗[/red] {models_message}")
+ console.print()
+
# Recommended actions
if not all_healthy:
# Check what can be auto-fixed
@@ -300,6 +319,35 @@ def display_health_report(
console.print()
+def check_models() -> tuple[bool, str]:
+ """Check if models are valid.
+
+ Returns:
+ Tuple of (is_healthy, message)
+ """
+ pipelex_hub = PipelexHub()
+ set_pipelex_hub(pipelex_hub)
+
+ # tools
+ try:
+ pipelex_hub.setup_config(config_cls=PipelexConfig)
+ except ValidationError as validation_error:
+ validation_error_msg = report_validation_error(category="config", validation_error=validation_error)
+ msg = f"Could not setup config because of: {validation_error_msg}"
+ raise PipelexConfigError(msg) from validation_error
+
+ log.configure(log_config=get_config().pipelex.log_config)
+
+ models_manager = ModelManager()
+ secrets_provider = EnvSecretsProvider()
+ try:
+ models_manager.setup(secrets_provider=secrets_provider)
+ models_manager.validate_model_deck()
+ except Exception as exc:
+ return False, f"Error checking models: {exc}"
+ return True, "Models are valid"
+
+
def doctor_cmd(
fix: bool = False,
) -> None:
@@ -309,142 +357,158 @@ def doctor_cmd(
fix: If True, offer to fix detected issues interactively
"""
console = Console()
-
+ do_doctor_cmd(console=console, fix=fix)
try:
- # Run health checks
- config_healthy, config_missing_count, config_message = check_config_files()
- telemetry_healthy, telemetry_message = check_telemetry_config()
- backends_healthy, backend_reports, backends_message = check_backend_credentials()
-
- # Display report
- display_health_report(
- console=console,
- config_healthy=config_healthy,
- config_message=config_message,
- config_missing_count=config_missing_count,
- telemetry_healthy=telemetry_healthy,
- telemetry_message=telemetry_message,
- backends_healthy=backends_healthy,
- backends_message=backends_message,
- backend_reports=backend_reports,
- )
+ do_doctor_cmd(console=console, fix=fix)
- all_healthy = config_healthy and telemetry_healthy and backends_healthy
+ except Exception as exc:
+ # Handle unexpected errors gracefully without printing traces
+ console.print()
+ console.print(f"[red]✗ Unexpected error: {exc!s}[/red]")
+ console.print()
+ console.print("[dim]If you need help:[/dim]")
+ console.print(" [cyan]https://docs.pipelex.com[/cyan] - Documentation")
+ console.print(" [cyan]https://go.pipelex.com/discord[/cyan] - Discord Community")
+ console.print()
+ sys.exit(1)
- # Exit code: 0 if healthy, 1 if issues found
- if all_healthy:
- sys.exit(0)
- # Determine what can be auto-fixed
- can_fix_config = not config_healthy and config_missing_count > 0
- can_fix_telemetry = not telemetry_healthy and "not found" in telemetry_message.lower()
- has_auto_fixable_issues = can_fix_config or can_fix_telemetry
+def do_doctor_cmd(
+ console: Console,
+ fix: bool = False,
+) -> None:
+ """Check Pipelex configuration health and suggest fixes.
- # Determine what requires manual fixes
- has_config_validation_error = not config_healthy and config_missing_count == 0
- has_telemetry_validation_error = not telemetry_healthy and "not found" not in telemetry_message.lower()
- has_backend_credential_issues = not backends_healthy and backend_reports
+ Args:
+ console: Rich Console instance for output
+ fix: If True, offer to fix detected issues interactively
+ """
+ # Run health checks
+ config_healthy, config_missing_count, config_message = check_config_files()
+ telemetry_healthy, telemetry_message = check_telemetry_config()
+ backends_healthy, backend_reports, backends_message = check_backend_credentials()
+ models_healthy, models_message = check_models()
+
+ # Display report
+ display_health_report(
+ console=console,
+ config_healthy=config_healthy,
+ config_message=config_message,
+ config_missing_count=config_missing_count,
+ telemetry_healthy=telemetry_healthy,
+ telemetry_message=telemetry_message,
+ backends_healthy=backends_healthy,
+ backends_message=backends_message,
+ backend_reports=backend_reports,
+ models_healthy=models_healthy,
+ models_message=models_message,
+ )
- # If --fix flag is provided, offer to fix auto-fixable issues
- if fix and has_auto_fixable_issues:
- console.print("[bold yellow]Interactive Fix Mode[/bold yellow]")
- console.print()
+ all_healthy = config_healthy and telemetry_healthy and backends_healthy and models_healthy
- # Fix missing config files
- if can_fix_config:
- if Confirm.ask(f"[bold]Install {config_missing_count} missing configuration file(s)?[/bold]", default=True):
- try:
- console.print()
- init_cmd(focus=InitFocus.CONFIG, reset=False, skip_confirmation=True)
- console.print("[green]✓[/green] Configuration files installed")
- except Exception as exc:
- console.print(f"[red]Failed to install configuration files: {exc!s}[/red]")
- console.print()
+ # Exit code: 0 if healthy, 1 if issues found
+ if all_healthy:
+ sys.exit(0)
- # Fix missing telemetry config
- if can_fix_telemetry:
- if Confirm.ask("[bold]Configure telemetry preferences?[/bold]", default=True):
- try:
- console.print()
- init_cmd(focus=InitFocus.TELEMETRY, reset=False, skip_confirmation=True)
- console.print("[green]✓[/green] Telemetry configured")
- except Exception as exc:
- console.print(f"[red]Failed to configure telemetry: {exc!s}[/red]")
- console.print()
+ # Determine what can be auto-fixed
+ can_fix_config = not config_healthy and config_missing_count > 0
+ can_fix_telemetry = not telemetry_healthy and "not found" in telemetry_message.lower()
+ has_auto_fixable_issues = can_fix_config or can_fix_telemetry
- # Handle issues that can't be auto-fixed
- if has_config_validation_error or has_telemetry_validation_error or has_backend_credential_issues:
- console.print("[bold yellow]Manual Fixes Required[/bold yellow]")
- console.print()
+ # Determine what requires manual fixes
+ has_config_validation_error = not config_healthy and config_missing_count == 0
+ has_telemetry_validation_error = not telemetry_healthy and "not found" not in telemetry_message.lower()
+ has_backend_credential_issues = not backends_healthy and backend_reports
- # Config validation errors
- if has_config_validation_error:
- console.print("[bold]Configuration validation error:[/bold]")
- console.print(f" {config_message}")
- console.print()
- console.print("You can fix this manually by editing [cyan].pipelex/pipelex.toml[/cyan]")
- console.print("or run [cyan]pipelex init config --reset[/cyan] to regenerate from template.")
- console.print()
+ # If --fix flag is provided, offer to fix auto-fixable issues
+ if fix and has_auto_fixable_issues:
+ console.print("[bold yellow]Interactive Fix Mode[/bold yellow]")
+ console.print()
- # Telemetry validation errors
- if has_telemetry_validation_error:
- console.print("[bold]Telemetry validation error:[/bold]")
- console.print(f" {telemetry_message}")
+ # Fix missing config files
+ if can_fix_config:
+ if Confirm.ask(f"[bold]Install {config_missing_count} missing configuration file(s)?[/bold]", default=True):
+ try:
+ console.print()
+ init_cmd(focus=InitFocus.CONFIG, reset=False, skip_confirmation=True)
+ console.print("[green]✓[/green] Configuration files installed")
+ except Exception as exc:
+ console.print(f"[red]Failed to install configuration files: {exc!s}[/red]")
console.print()
- console.print("You can fix this manually by editing [cyan].pipelex/telemetry.toml[/cyan]")
- console.print("or run [cyan]pipelex init telemetry --reset[/cyan] to regenerate from template.")
+
+ # Fix missing telemetry config
+ if can_fix_telemetry:
+ if Confirm.ask("[bold]Configure telemetry preferences?[/bold]", default=True):
+ try:
+ console.print()
+ init_cmd(focus=InitFocus.TELEMETRY, reset=False, skip_confirmation=True)
+ console.print("[green]✓[/green] Telemetry configured")
+ except Exception as exc:
+ console.print(f"[red]Failed to configure telemetry: {exc!s}[/red]")
console.print()
- # Backend credentials
- if has_backend_credential_issues:
- all_missing_vars: set[str] = set()
- for backend_report in backend_reports.values():
- if not backend_report.all_credentials_valid:
- all_missing_vars.update(backend_report.missing_vars)
+ # Handle issues that can't be auto-fixed
+ if has_config_validation_error or has_telemetry_validation_error or has_backend_credential_issues:
+ console.print("[bold yellow]Manual Fixes Required[/bold yellow]")
+ console.print()
- if all_missing_vars:
- console.print("[bold]Backend credentials:[/bold]")
- console.print()
- console.print("Set the following environment variables:")
- console.print()
+ # Config validation errors
+ if has_config_validation_error:
+ console.print("[bold]Configuration validation error:[/bold]")
+ console.print(f" {config_message}")
+ console.print()
+ console.print("You can fix this manually by editing [cyan].pipelex/pipelex.toml[/cyan]")
+ console.print("or run [cyan]pipelex init config --reset[/cyan] to regenerate from template.")
+ console.print()
- # Show .env file syntax first
- console.print("[dim]# In your .env file:[/dim]")
- for var_name in sorted(all_missing_vars):
- console.print(f"{var_name}=[yellow]your_value_here[/yellow]")
- console.print()
+ # Telemetry validation errors
+ if has_telemetry_validation_error:
+ console.print("[bold]Telemetry validation error:[/bold]")
+ console.print(f" {telemetry_message}")
+ console.print()
+ console.print("You can fix this manually by editing [cyan].pipelex/telemetry.toml[/cyan]")
+ console.print("or run [cyan]pipelex init telemetry --reset[/cyan] to regenerate from template.")
+ console.print()
- # Show shell syntax for different platforms
- console.print("[dim]# Or in your shell:[/dim]")
- console.print()
+ # Backend credentials
+ if has_backend_credential_issues:
+ all_missing_vars: set[str] = set()
+ for backend_report in backend_reports.values():
+ if not backend_report.all_credentials_valid:
+ all_missing_vars.update(backend_report.missing_vars)
- # Linux/MacOS
- console.print("[dim]# Linux/MacOS[/dim]")
- for var_name in sorted(all_missing_vars):
- console.print(f"export {var_name}=[yellow]your_value_here[/yellow]")
- console.print()
+ if all_missing_vars:
+ console.print("[bold]Backend credentials:[/bold]")
+ console.print()
+ console.print("Set the following environment variables:")
+ console.print()
- # Windows PowerShell
- console.print("[dim]# Windows PowerShell[/dim]")
- for var_name in sorted(all_missing_vars):
- console.print(f'$env:{var_name}="[yellow]your_value_here[/yellow]"')
- console.print()
+ # Show .env file syntax first
+ console.print("[dim]# In your .env file:[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f"{var_name}=[yellow]your_value_here[/yellow]")
+ console.print()
- # Windows CMD
- console.print("[dim]# Windows CMD[/dim]")
- for var_name in sorted(all_missing_vars):
- console.print(f"set {var_name}=[yellow]your_value_here[/yellow]")
- console.print()
+ # Show shell syntax for different platforms
+ console.print("[dim]# Or in your shell:[/dim]")
+ console.print()
- sys.exit(1)
+ # Linux/MacOS
+ console.print("[dim]# Linux/MacOS[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f"export {var_name}=[yellow]your_value_here[/yellow]")
+ console.print()
- except Exception as exc:
- # Handle unexpected errors gracefully without printing traces
- console.print()
- console.print(f"[red]✗ Unexpected error: {exc!s}[/red]")
- console.print()
- console.print("[dim]If you need help:[/dim]")
- console.print(" [cyan]https://docs.pipelex.com[/cyan] - Documentation")
- console.print(" [cyan]https://go.pipelex.com/discord[/cyan] - Discord Community")
- console.print()
- sys.exit(1)
+ # Windows PowerShell
+ console.print("[dim]# Windows PowerShell[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f'$env:{var_name}="[yellow]your_value_here[/yellow]"')
+ console.print()
+
+ # Windows CMD
+ console.print("[dim]# Windows CMD[/dim]")
+ for var_name in sorted(all_missing_vars):
+ console.print(f"set {var_name}=[yellow]your_value_here[/yellow]")
+ console.print()
+
+ sys.exit(1)
diff --git a/pipelex/cli/commands/init/__init__.py b/pipelex/cli/commands/init/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/cli/commands/init/backends.py b/pipelex/cli/commands/init/backends.py
new file mode 100644
index 000000000..b07849f43
--- /dev/null
+++ b/pipelex/cli/commands/init/backends.py
@@ -0,0 +1,100 @@
+"""Backend configuration logic for the init command."""
+
+import os
+from typing import Any
+
+from rich.console import Console
+
+from pipelex.cli.commands.init.ui.backends_ui import (
+ build_backend_selection_panel,
+ display_selected_backends,
+ get_backend_options_from_toml,
+ get_currently_enabled_backends,
+ prompt_backend_indices,
+)
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.system.configuration.config_loader import config_manager
+from pipelex.tools.misc.file_utils import path_exists
+from pipelex.tools.misc.toml_utils import load_toml_from_path, load_toml_with_tomlkit, save_toml_to_path
+
+
+def update_backends_in_toml(toml_doc: Any, selected_indices: list[int], backend_options: list[tuple[str, str]]) -> None:
+ """Update the backends.toml document with selected backends.
+
+ Args:
+ toml_doc: The TOML document to update.
+ selected_indices: List of backend indices to enable.
+ backend_options: List of available backend options.
+ """
+ selected_backend_keys = {backend_options[idx][0] for idx in selected_indices}
+
+ # Disable all backends first (except internal)
+ for backend_key in toml_doc:
+ if backend_key != "internal" and backend_key in toml_doc:
+ backend_section = toml_doc[backend_key]
+ # Set enabled field based on selection (works with tomlkit's special types)
+ backend_section["enabled"] = backend_key in selected_backend_keys # type: ignore[index]
+
+
+def get_selected_backend_keys(backends_toml_path: str) -> list[str]:
+ """Extract the list of enabled backend keys from backends.toml.
+
+ Args:
+ backends_toml_path: Path to the backends.toml file.
+
+ Returns:
+ List of backend keys that are enabled (excluding 'internal').
+ """
+ selected_backends: list[str] = []
+
+ if not path_exists(backends_toml_path):
+ return selected_backends
+
+ toml_doc = load_toml_from_path(backends_toml_path)
+
+ for backend_key in toml_doc:
+ if backend_key != "internal":
+ backend_section = toml_doc[backend_key]
+ if isinstance(backend_section, dict):
+ if backend_section.get("enabled", False) is True: # pyright: ignore[reportUnknownMemberType]
+ selected_backends.append(backend_key)
+
+ return selected_backends
+
+
+def customize_backends_config() -> None:
+ """Interactively customize which inference backends are enabled in backends.toml."""
+ console = Console()
+ backends_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "backends.toml")
+ template_backends_path = os.path.join(str(get_kit_configs_dir()), "inference", "backends.toml")
+
+ if not path_exists(backends_toml_path):
+ console.print("[yellow]⚠ Warning: backends.toml not found, skipping backend customization[/yellow]")
+ return
+
+ try:
+ # Get backend options from template and existing config
+ existing_path = backends_toml_path if path_exists(backends_toml_path) else None
+ backend_options = get_backend_options_from_toml(template_backends_path, existing_path)
+
+ # Get currently enabled backends to show user their current selection
+ currently_enabled = get_currently_enabled_backends(backends_toml_path, backend_options)
+
+ # Load the backends.toml file
+ toml_doc = load_toml_with_tomlkit(backends_toml_path)
+ console.print()
+
+ # UI: Display panel and get user selection
+ console.print(build_backend_selection_panel(backend_options, currently_enabled))
+ selected_indices = prompt_backend_indices(console, backend_options, currently_enabled)
+
+ # Business logic: Update TOML
+ update_backends_in_toml(toml_doc, selected_indices, backend_options)
+ save_toml_to_path(toml_doc, backends_toml_path)
+
+ # UI: Display confirmation
+ display_selected_backends(console, selected_indices, backend_options)
+
+ except Exception as exc:
+ console.print(f"[yellow]⚠ Warning: Failed to customize backends: {exc}[/yellow]")
+ console.print("[dim]You can manually edit .pipelex/inference/backends.toml later[/dim]")
diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init/command.py
similarity index 52%
rename from pipelex/cli/commands/init_cmd.py
rename to pipelex/cli/commands/init/command.py
index 7a7e198dc..093fddc99 100644
--- a/pipelex/cli/commands/init_cmd.py
+++ b/pipelex/cli/commands/init/command.py
@@ -1,217 +1,62 @@
+"""Main command orchestration for the init command."""
+
import os
-import shutil
-from typing import Any
import typer
from rich.console import Console
from rich.prompt import Confirm
-from pipelex.cli.commands.init_ui import (
- InitFocus,
- build_backend_selection_panel,
- build_initialization_panel,
- build_telemetry_selection_panel,
- display_already_configured_message,
- display_selected_backends,
- get_backend_options_from_toml,
- get_currently_enabled_backends,
- prompt_backend_indices,
- prompt_telemetry_mode,
-)
-from pipelex.exceptions import PipelexCLIError
-from pipelex.kit.paths import get_configs_dir
+from pipelex.cli.commands.init.backends import customize_backends_config, get_selected_backend_keys
+from pipelex.cli.commands.init.config_files import init_config
+from pipelex.cli.commands.init.routing import customize_routing_profile
+from pipelex.cli.commands.init.telemetry import setup_telemetry
+from pipelex.cli.commands.init.ui.general_ui import build_initialization_panel, display_already_configured_message
+from pipelex.cli.commands.init.ui.types import InitFocus
from pipelex.system.configuration.config_loader import config_manager
-from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryMode
+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME
from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract
from pipelex.tools.misc.file_utils import path_exists
-from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
-
-
-def update_backends_in_toml(toml_doc: Any, selected_indices: list[int], backend_options: list[tuple[str, str]]) -> None:
- """Update the backends.toml document with selected backends.
-
- Args:
- toml_doc: The TOML document to update.
- selected_indices: List of backend indices to enable.
- backend_options: List of available backend options.
- """
- selected_backend_keys = {backend_options[idx][0] for idx in selected_indices}
-
- # Disable all backends first (except internal)
- for backend_key in toml_doc:
- if backend_key != "internal" and backend_key in toml_doc:
- backend_section = toml_doc[backend_key]
- # Set enabled field based on selection (works with tomlkit's special types)
- backend_section["enabled"] = backend_key in selected_backend_keys # type: ignore[index]
-
-
-def customize_backends_config() -> None:
- """Interactively customize which inference backends are enabled in backends.toml."""
- console = Console()
- backends_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "backends.toml")
- template_backends_path = os.path.join(str(get_configs_dir()), "inference", "backends.toml")
-
- if not path_exists(backends_toml_path):
- console.print("[yellow]⚠ Warning: backends.toml not found, skipping backend customization[/yellow]")
- return
-
- try:
- # Get backend options from template and existing config
- existing_path = backends_toml_path if path_exists(backends_toml_path) else None
- backend_options = get_backend_options_from_toml(template_backends_path, existing_path)
-
- # Get currently enabled backends to show user their current selection
- currently_enabled = get_currently_enabled_backends(backends_toml_path, backend_options)
-
- # Load the backends.toml file
- toml_doc = load_toml_with_tomlkit(backends_toml_path)
- console.print()
-
- # UI: Display panel and get user selection
- console.print(build_backend_selection_panel(backend_options, currently_enabled))
- selected_indices = prompt_backend_indices(console, backend_options, currently_enabled)
-
- # Business logic: Update TOML
- update_backends_in_toml(toml_doc, selected_indices, backend_options)
- save_toml_to_path(toml_doc, backends_toml_path)
-
- # UI: Display confirmation
- display_selected_backends(console, selected_indices, backend_options)
-
- except Exception as exc:
- console.print(f"[yellow]⚠ Warning: Failed to customize backends: {exc}[/yellow]")
- console.print("[dim]You can manually edit .pipelex/inference/backends.toml later[/dim]")
-
-
-def init_config(reset: bool = False, dry_run: bool = False) -> int:
- """Initialize pipelex configuration in the .pipelex directory. Does not install telemetry, just the main config dans inference backends.
-
- Args:
- reset: Whether to overwrite existing files.
- dry_run: Whether to only print the files that would be copied, without actually copying them.
-
- Returns:
- The number of files copied.
- """
- config_template_dir = str(get_configs_dir())
- target_config_dir = config_manager.pipelex_config_dir
-
- os.makedirs(target_config_dir, exist_ok=True)
-
- try:
- copied_files: list[str] = []
- existing_files: list[str] = []
-
- def copy_directory_structure(src_dir: str, dst_dir: str, relative_path: str = "", dry_run: bool = False) -> None:
- """Recursively copy directory structure, handling existing files."""
- for item in os.listdir(src_dir):
- src_item = os.path.join(src_dir, item)
- dst_item = os.path.join(dst_dir, item)
- relative_item = os.path.join(relative_path, item) if relative_path else item
-
- # Skip telemetry.toml - it will be created when user is prompted
- if item == TELEMETRY_CONFIG_FILE_NAME:
- continue
-
- if os.path.isdir(src_item):
- if not dry_run:
- os.makedirs(dst_item, exist_ok=True)
- copy_directory_structure(src_item, dst_item, relative_item, dry_run)
- elif os.path.exists(dst_item) and not reset:
- existing_files.append(relative_item)
- else:
- if not dry_run:
- shutil.copy2(src_item, dst_item)
- copied_files.append(relative_item)
-
- copy_directory_structure(src_dir=config_template_dir, dst_dir=target_config_dir, dry_run=dry_run)
-
- if dry_run:
- return len(copied_files)
-
- # Report results
- if copied_files:
- typer.echo(f"✅ Copied {len(copied_files)} files to {target_config_dir}:")
- for file in sorted(copied_files):
- typer.echo(f" • {file}")
-
- if existing_files:
- typer.echo(f"ℹ️ Skipped {len(existing_files)} existing files (use --reset to overwrite):")
- for file in sorted(existing_files):
- typer.echo(f" • {file}")
-
- if not copied_files and not existing_files:
- typer.echo(f"✅ Configuration directory {target_config_dir} is already up to date")
-
- except Exception as exc:
- msg = f"Failed to initialize configuration: {exc}"
- raise PipelexCLIError(msg) from exc
-
- return len(copied_files)
-
-
-def setup_telemetry(console: Console, telemetry_config_path: str) -> TelemetryMode:
- """Set up telemetry configuration interactively.
-
- Args:
- console: Rich Console instance for user interaction.
- telemetry_config_path: Path to save the telemetry configuration.
-
- Returns:
- The selected TelemetryMode.
-
- Raises:
- typer.Exit: If user chooses to quit.
- """
- console.print()
- console.print(build_telemetry_selection_panel())
-
- telemetry_mode = prompt_telemetry_mode(console)
-
- # Save telemetry config
- template_path = os.path.join(str(get_configs_dir()), TELEMETRY_CONFIG_FILE_NAME)
- toml_doc = load_toml_with_tomlkit(template_path)
- toml_doc["telemetry_mode"] = telemetry_mode
- save_toml_to_path(toml_doc, telemetry_config_path)
-
- console.print(f"\n[green]✓[/green] Telemetry mode set to: [bold cyan]{telemetry_mode}[/bold cyan]")
-
- return telemetry_mode
def determine_needs(
reset: bool,
check_config: bool,
check_inference: bool,
+ check_routing: bool,
check_telemetry: bool,
backends_toml_path: str,
+ routing_profiles_toml_path: str,
telemetry_config_path: str,
-) -> tuple[bool, bool, bool]:
+) -> tuple[bool, bool, bool, bool]:
"""Determine what needs to be initialized based on current state.
Args:
reset: Whether this is a reset operation.
check_config: Whether to check config files.
check_inference: Whether to check inference setup.
+ check_routing: Whether to check routing setup.
check_telemetry: Whether to check telemetry setup.
backends_toml_path: Path to backends.toml file.
+ routing_profiles_toml_path: Path to routing_profiles.toml file.
telemetry_config_path: Path to telemetry config file.
Returns:
- Tuple of (needs_config, needs_inference, needs_telemetry) booleans.
+ Tuple of (needs_config, needs_inference, needs_routing, needs_telemetry) booleans.
"""
nb_missing_config_files = init_config(reset=False, dry_run=True) if check_config else 0
needs_config = check_config and (nb_missing_config_files > 0 or reset)
needs_inference = check_inference and (not path_exists(backends_toml_path) or reset)
+ needs_routing = check_routing and (not path_exists(routing_profiles_toml_path) or reset)
needs_telemetry = check_telemetry and (not path_exists(telemetry_config_path) or reset)
- return needs_config, needs_inference, needs_telemetry
+ return needs_config, needs_inference, needs_routing, needs_telemetry
def handle_already_configured(
focus: InitFocus,
console: Console,
backends_toml_path: str,
+ routing_profiles_toml_path: str,
telemetry_config_path: str,
) -> bool:
"""Handle the case when everything is already configured.
@@ -220,6 +65,7 @@ def handle_already_configured(
focus: The initialization focus area.
console: Rich Console instance for output.
backends_toml_path: Path to backends.toml file.
+ routing_profiles_toml_path: Path to routing_profiles.toml file.
telemetry_config_path: Path to telemetry config file.
Returns:
@@ -228,6 +74,7 @@ def handle_already_configured(
# Map focus to config path for display
config_path_map = {
InitFocus.INFERENCE: backends_toml_path,
+ InitFocus.ROUTING: routing_profiles_toml_path,
InitFocus.TELEMETRY: telemetry_config_path,
InitFocus.CONFIG: ".pipelex/",
}
@@ -236,26 +83,28 @@ def handle_already_configured(
return display_already_configured_message(focus, console, config_path)
-def update_needs_for_reconfigure(focus: InitFocus) -> tuple[bool, bool, bool]:
+def update_needs_for_reconfigure(focus: InitFocus) -> tuple[bool, bool, bool, bool]:
"""Update needs flags when user wants to reconfigure.
Args:
focus: The initialization focus area.
Returns:
- Tuple of (needs_config, needs_inference, needs_telemetry) booleans.
+ Tuple of (needs_config, needs_inference, needs_routing, needs_telemetry) booleans.
"""
needs_config = focus == InitFocus.CONFIG
needs_inference = focus == InitFocus.INFERENCE
+ needs_routing = focus == InitFocus.ROUTING
needs_telemetry = focus == InitFocus.TELEMETRY
- return needs_config, needs_inference, needs_telemetry
+ return needs_config, needs_inference, needs_routing, needs_telemetry
def confirm_initialization(
console: Console,
needs_config: bool,
needs_inference: bool,
+ needs_routing: bool,
needs_telemetry: bool,
reset: bool,
focus: InitFocus,
@@ -266,6 +115,7 @@ def confirm_initialization(
console: Rich Console instance for user interaction.
needs_config: Whether config initialization is needed.
needs_inference: Whether inference setup is needed.
+ needs_routing: Whether routing setup is needed.
needs_telemetry: Whether telemetry setup is needed.
reset: Whether this is a reset operation.
focus: The initialization focus area.
@@ -277,15 +127,15 @@ def confirm_initialization(
typer.Exit: If user cancels initialization.
"""
console.print()
- console.print(build_initialization_panel(needs_config, needs_inference, needs_telemetry, reset))
+ console.print(build_initialization_panel(needs_config, needs_inference, needs_routing, needs_telemetry, reset))
if not Confirm.ask("[bold]Continue with initialization?[/bold]", default=True):
console.print("\n[yellow]Initialization cancelled.[/yellow]")
- if needs_config or needs_inference or needs_telemetry:
+ if needs_config or needs_inference or needs_routing or needs_telemetry:
match focus:
case InitFocus.ALL:
init_cmd_str = "pipelex init"
- case InitFocus.CONFIG | InitFocus.INFERENCE | InitFocus.TELEMETRY:
+ case InitFocus.CONFIG | InitFocus.INFERENCE | InitFocus.ROUTING | InitFocus.TELEMETRY:
init_cmd_str = f"pipelex init {focus}"
console.print(f"[dim]You can initialize later by running:[/dim] [cyan]{init_cmd_str}[/cyan]")
console.print()
@@ -298,9 +148,11 @@ def execute_initialization(
console: Console,
needs_config: bool,
needs_inference: bool,
+ needs_routing: bool,
needs_telemetry: bool,
reset: bool,
check_inference: bool,
+ check_routing: bool,
backends_toml_path: str,
telemetry_config_path: str,
):
@@ -310,9 +162,11 @@ def execute_initialization(
console: Rich Console instance for output.
needs_config: Whether to initialize config files.
needs_inference: Whether to set up inference backends.
+ needs_routing: Whether to set up routing profiles.
needs_telemetry: Whether to set up telemetry.
reset: Whether this is a reset operation.
check_inference: Whether inference was in focus.
+ check_routing: Whether routing was in focus.
backends_toml_path: Path to backends.toml file.
telemetry_config_path: Path to telemetry config file.
@@ -337,6 +191,21 @@ def execute_initialization(
console.print()
customize_backends_config()
+ # Automatically set up routing after backends (unless routing is the specific focus)
+ if not check_routing:
+ selected_backend_keys = get_selected_backend_keys(backends_toml_path)
+ if selected_backend_keys:
+ customize_routing_profile(selected_backend_keys)
+
+ # Step 2.5: Set up routing profile if specifically requested
+ if needs_routing:
+ console.print()
+ selected_backend_keys = get_selected_backend_keys(backends_toml_path)
+ if selected_backend_keys:
+ customize_routing_profile(selected_backend_keys)
+ else:
+ console.print("[yellow]⚠ Warning: No backends enabled. Please run 'pipelex init inference' first.[/yellow]")
+
# Step 3: Set up telemetry if needed
if needs_telemetry:
telemetry_mode = setup_telemetry(console, telemetry_config_path)
@@ -351,10 +220,10 @@ def init_cmd(
skip_confirmation: bool = False,
silent: bool = False,
):
- """Initialize Pipelex configuration, inference backends, and telemetry if needed, in a unified flow.
+ """Initialize Pipelex configuration, inference backends, routing, and telemetry if needed, in a unified flow.
Args:
- focus: What to initialize - 'config', 'inference', 'telemetry', or 'all' (default)
+ focus: What to initialize - 'config', 'inference', 'routing', 'telemetry', or 'all' (default)
reset: Whether to reset/overwrite existing files
skip_confirmation: If True, skip the confirmation prompt (used when called from doctor --fix)
silent: If True, suppress all output when everything is already configured
@@ -363,19 +232,23 @@ def init_cmd(
pipelex_config_dir = config_manager.pipelex_config_dir
telemetry_config_path = os.path.join(pipelex_config_dir, TELEMETRY_CONFIG_FILE_NAME)
backends_toml_path = os.path.join(pipelex_config_dir, "inference", "backends.toml")
+ routing_profiles_toml_path = os.path.join(pipelex_config_dir, "inference", "routing_profiles.toml")
# Determine what to check based on focus parameter
check_config = focus in (InitFocus.ALL, InitFocus.CONFIG)
check_inference = focus in (InitFocus.ALL, InitFocus.INFERENCE)
+ check_routing = focus == InitFocus.ROUTING
check_telemetry = focus in (InitFocus.ALL, InitFocus.TELEMETRY)
# Check what needs to be initialized
- needs_config, needs_inference, needs_telemetry = determine_needs(
+ needs_config, needs_inference, needs_routing, needs_telemetry = determine_needs(
reset=reset,
check_config=check_config,
check_inference=check_inference,
+ check_routing=check_routing,
check_telemetry=check_telemetry,
backends_toml_path=backends_toml_path,
+ routing_profiles_toml_path=routing_profiles_toml_path,
telemetry_config_path=telemetry_config_path,
)
@@ -383,14 +256,14 @@ def init_cmd(
user_already_confirmed = False
# If nothing needs to be done, handle based on focus
- if not needs_config and not needs_inference and not needs_telemetry:
+ if not needs_config and not needs_inference and not needs_routing and not needs_telemetry:
# In silent mode, just return without any output
if silent:
return
- if handle_already_configured(focus, console, backends_toml_path, telemetry_config_path):
+ if handle_already_configured(focus, console, backends_toml_path, routing_profiles_toml_path, telemetry_config_path):
# User wants to reconfigure
- needs_config, needs_inference, needs_telemetry = update_needs_for_reconfigure(focus)
+ needs_config, needs_inference, needs_routing, needs_telemetry = update_needs_for_reconfigure(focus)
user_already_confirmed = True
else:
# User doesn't want to reconfigure, exit
@@ -405,6 +278,7 @@ def init_cmd(
console=console,
needs_config=needs_config,
needs_inference=needs_inference,
+ needs_routing=needs_routing,
needs_telemetry=needs_telemetry,
reset=reset,
focus=focus,
@@ -418,9 +292,11 @@ def init_cmd(
console=console,
needs_config=needs_config,
needs_inference=needs_inference,
+ needs_routing=needs_routing,
needs_telemetry=needs_telemetry,
reset=reset,
check_inference=check_inference,
+ check_routing=check_routing,
backends_toml_path=backends_toml_path,
telemetry_config_path=telemetry_config_path,
)
diff --git a/pipelex/cli/commands/init/config_files.py b/pipelex/cli/commands/init/config_files.py
new file mode 100644
index 000000000..c565bbd86
--- /dev/null
+++ b/pipelex/cli/commands/init/config_files.py
@@ -0,0 +1,78 @@
+"""Configuration files management for the init command."""
+
+import os
+import shutil
+
+import typer
+
+from pipelex.exceptions import PipelexCLIError
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.system.configuration.config_loader import config_manager
+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME
+
+
+def init_config(reset: bool = False, dry_run: bool = False) -> int:
+ """Initialize pipelex configuration in the .pipelex directory. Does not install telemetry, just the main config dans inference backends.
+
+ Args:
+ reset: Whether to overwrite existing files.
+ dry_run: Whether to only print the files that would be copied, without actually copying them.
+
+ Returns:
+ The number of files copied.
+ """
+ config_template_dir = str(get_kit_configs_dir())
+ target_config_dir = config_manager.pipelex_config_dir
+
+ os.makedirs(target_config_dir, exist_ok=True)
+
+ try:
+ copied_files: list[str] = []
+ existing_files: list[str] = []
+
+ def copy_directory_structure(src_dir: str, dst_dir: str, relative_path: str = "", dry_run: bool = False) -> None:
+ """Recursively copy directory structure, handling existing files."""
+ for item in os.listdir(src_dir):
+ src_item = os.path.join(src_dir, item)
+ dst_item = os.path.join(dst_dir, item)
+ relative_item = os.path.join(relative_path, item) if relative_path else item
+
+ # Skip telemetry.toml - it will be created when user is prompted
+ if item == TELEMETRY_CONFIG_FILE_NAME:
+ continue
+
+ if os.path.isdir(src_item):
+ if not dry_run:
+ os.makedirs(dst_item, exist_ok=True)
+ copy_directory_structure(src_item, dst_item, relative_item, dry_run)
+ elif os.path.exists(dst_item) and not reset:
+ existing_files.append(relative_item)
+ else:
+ if not dry_run:
+ shutil.copy2(src_item, dst_item)
+ copied_files.append(relative_item)
+
+ copy_directory_structure(src_dir=config_template_dir, dst_dir=target_config_dir, dry_run=dry_run)
+
+ if dry_run:
+ return len(copied_files)
+
+ # Report results
+ if copied_files:
+ typer.echo(f"✅ Copied {len(copied_files)} files to {target_config_dir}:")
+ for file in sorted(copied_files):
+ typer.echo(f" • {file}")
+
+ if existing_files:
+ typer.echo(f"ℹ️ Skipped {len(existing_files)} existing files (use --reset to overwrite):")
+ for file in sorted(existing_files):
+ typer.echo(f" • {file}")
+
+ if not copied_files and not existing_files:
+ typer.echo(f"✅ Configuration directory {target_config_dir} is already up to date")
+
+ except Exception as exc:
+ msg = f"Failed to initialize configuration: {exc}"
+ raise PipelexCLIError(msg) from exc
+
+ return len(copied_files)
diff --git a/pipelex/cli/commands/init/routing.py b/pipelex/cli/commands/init/routing.py
new file mode 100644
index 000000000..cb0504e17
--- /dev/null
+++ b/pipelex/cli/commands/init/routing.py
@@ -0,0 +1,153 @@
+"""Routing profile configuration logic for the init command."""
+
+import os
+from typing import Any
+
+from rich.console import Console
+from rich.prompt import Confirm
+from tomlkit import table
+
+from pipelex.cli.commands.init.ui.backends_ui import get_backend_options_from_toml
+from pipelex.cli.commands.init.ui.routing_ui import (
+ build_fallback_order_panel,
+ build_primary_backend_panel,
+ display_routing_profile_result,
+ prompt_fallback_order,
+ prompt_primary_backend,
+)
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.system.configuration.config_loader import config_manager
+from pipelex.tools.misc.file_utils import path_exists
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
+
+
+def customize_routing_profile(selected_backend_keys: list[str]) -> None:
+ """Interactively customize routing profile based on selected backends.
+
+ Args:
+ selected_backend_keys: List of backend keys that are enabled.
+ """
+ console = Console()
+ routing_profiles_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "routing_profiles.toml")
+ template_routing_path = os.path.join(str(get_kit_configs_dir()), "inference", "routing_profiles.toml")
+ backends_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "backends.toml")
+
+ if not path_exists(routing_profiles_toml_path):
+ console.print("[yellow]⚠ Warning: routing_profiles.toml not found, skipping routing customization[/yellow]")
+ return
+
+ try:
+ # Load template for reference (to check if profiles exist)
+ template_toml = load_toml_with_tomlkit(template_routing_path)
+
+ # Load the user's routing_profiles.toml file
+ toml_doc = load_toml_with_tomlkit(routing_profiles_toml_path)
+
+ # Get backend options for display names
+ template_backends_path = os.path.join(str(get_kit_configs_dir()), "inference", "backends.toml")
+ backend_options = get_backend_options_from_toml(template_backends_path, backends_toml_path)
+
+ # Case 1: pipelex_inference is enabled - keep pipelex_first
+ if "pipelex_inference" in selected_backend_keys:
+ toml_doc["active"] = "pipelex_first"
+ save_toml_to_path(toml_doc, routing_profiles_toml_path)
+ display_routing_profile_result(console, "pipelex_first", created=False)
+ return
+
+ # Case 2: Only one backend selected - use all_{backend_key} profile
+ if len(selected_backend_keys) == 1:
+ backend_key = selected_backend_keys[0]
+ profile_name = f"all_{backend_key}"
+
+ # Check if profile exists in template
+ template_profiles: dict[str, Any] = template_toml.get("profiles") or {} # type: ignore[assignment]
+ profile_exists = profile_name in template_profiles
+
+ if not profile_exists:
+ # Ask for confirmation to create the profile
+ console.print()
+ console.print(f"[yellow]Profile {profile_name!r} does not exist in the template.[/yellow]")
+ if not Confirm.ask("[bold]Would you like to create it?[/bold]", default=True):
+ console.print("[dim]Keeping current routing profile configuration.[/dim]")
+ return
+
+ # Create the profile
+ if "profiles" not in toml_doc:
+ toml_doc["profiles"] = {}
+
+ # Use a dict for the profile data
+ profile_data = {
+ "description": f"Use {backend_key} backend for all its supported models",
+ "default": backend_key,
+ }
+ toml_doc["profiles"][profile_name] = profile_data # type: ignore[index]
+
+ # Set as active profile
+ toml_doc["active"] = profile_name
+ save_toml_to_path(toml_doc, routing_profiles_toml_path)
+ display_routing_profile_result(console, profile_name, created=not profile_exists)
+ return
+
+ # Case 3: Multiple backends (no pipelex_inference) - create/update custom_routing profile
+ console.print()
+ console.print("[cyan]Setting up routing for multiple backends...[/cyan]")
+ console.print()
+
+ # Show panel and prompt for primary backend
+ console.print(build_primary_backend_panel(selected_backend_keys, backend_options))
+ primary_backend = prompt_primary_backend(console, selected_backend_keys)
+
+ # Prompt for fallback order if there are 2+ remaining backends
+ remaining_backends = [b for b in selected_backend_keys if b != primary_backend]
+ fallback_order: list[str] | None = None
+
+ if len(remaining_backends) >= 2:
+ console.print()
+ console.print(build_fallback_order_panel(remaining_backends, backend_options))
+ # Get the ordered remaining backends from user (default keeps current order)
+ ordered_remaining = prompt_fallback_order(console, remaining_backends, backend_options)
+ fallback_order = [primary_backend, *ordered_remaining]
+ elif len(remaining_backends) == 1:
+ # Only one remaining backend - set fallback_order with primary first
+ fallback_order = [primary_backend, *remaining_backends]
+ else:
+ # Only primary backend selected - no fallback needed
+ fallback_order = None
+
+ # Create or update custom_routing profile
+ if "profiles" not in toml_doc:
+ toml_doc["profiles"] = {}
+
+ # Insert custom_routing at the beginning of profiles (after standard profiles if they exist)
+ # We'll create a new profiles dict with custom_routing first
+ new_profiles = table()
+
+ # Create custom_routing profile with nested routes table
+ custom_routing_profile = table()
+ custom_routing_profile["description"] = "Custom routing"
+ custom_routing_profile["default"] = primary_backend
+ if fallback_order:
+ custom_routing_profile["fallback_order"] = fallback_order
+ custom_routing_profile["routes"] = table()
+ new_profiles["custom_routing"] = custom_routing_profile
+
+ # Copy existing profiles after custom_routing
+ existing_profiles: dict[str, Any] = toml_doc.get("profiles") or {} # type: ignore[assignment]
+ for profile_key, profile_value in existing_profiles.items(): # type: ignore[union-attr]
+ if profile_key != "custom_routing":
+ new_profiles[profile_key] = profile_value
+
+ toml_doc["profiles"] = new_profiles # type: ignore[assignment]
+ toml_doc["active"] = "custom_routing"
+
+ save_toml_to_path(toml_doc, routing_profiles_toml_path)
+ display_routing_profile_result(console, "custom_routing", created=True)
+
+ # Inform user about customization options
+ console.print("[dim]You can find your custom routing profile in:[/dim]")
+ console.print(f"[dim] {routing_profiles_toml_path}[/dim]")
+ console.print("[dim]You can further customize which models get used on which backend by editing the routes section.[/dim]")
+
+ except Exception as exc:
+ console.print(f"[yellow]⚠ Warning: Failed to customize routing profile: {exc}[/yellow]")
+ console.print("[dim]You can manually edit .pipelex/inference/routing_profiles.toml later[/dim]")
diff --git a/pipelex/cli/commands/init/telemetry.py b/pipelex/cli/commands/init/telemetry.py
new file mode 100644
index 000000000..cfe26ea2e
--- /dev/null
+++ b/pipelex/cli/commands/init/telemetry.py
@@ -0,0 +1,39 @@
+"""Telemetry configuration logic for the init command."""
+
+import os
+
+from rich.console import Console
+
+from pipelex.cli.commands.init.ui.telemetry_ui import build_telemetry_selection_panel, prompt_telemetry_mode
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryMode
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
+
+
+def setup_telemetry(console: Console, telemetry_config_path: str) -> TelemetryMode:
+ """Set up telemetry configuration interactively.
+
+ Args:
+ console: Rich Console instance for user interaction.
+ telemetry_config_path: Path to save the telemetry configuration.
+
+ Returns:
+ The selected TelemetryMode.
+
+ Raises:
+ typer.Exit: If user chooses to quit.
+ """
+ console.print()
+ console.print(build_telemetry_selection_panel())
+
+ telemetry_mode = prompt_telemetry_mode(console)
+
+ # Save telemetry config
+ template_path = os.path.join(str(get_kit_configs_dir()), TELEMETRY_CONFIG_FILE_NAME)
+ toml_doc = load_toml_with_tomlkit(template_path)
+ toml_doc["telemetry_mode"] = telemetry_mode
+ save_toml_to_path(toml_doc, telemetry_config_path)
+
+ console.print(f"\n[green]✓[/green] Telemetry mode set to: [bold cyan]{telemetry_mode}[/bold cyan]")
+
+ return telemetry_mode
diff --git a/pipelex/cli/commands/init/ui/__init__.py b/pipelex/cli/commands/init/ui/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/cli/commands/init_ui.py b/pipelex/cli/commands/init/ui/backends_ui.py
similarity index 55%
rename from pipelex/cli/commands/init_ui.py
rename to pipelex/cli/commands/init/ui/backends_ui.py
index 309d15724..c7fa48774 100644
--- a/pipelex/cli/commands/init_ui.py
+++ b/pipelex/cli/commands/init/ui/backends_ui.py
@@ -1,23 +1,15 @@
-"""UI components for the init command.
-
-This module contains all user interface logic for the Pipelex initialization process,
-including prompts, panels, and user input validation.
-"""
-
-from __future__ import annotations
+"""UI components for backend configuration in the init command."""
import typer
from rich.console import Console, Group
from rich.panel import Panel
-from rich.prompt import Confirm, Prompt
+from rich.prompt import Prompt
from rich.table import Table
from rich.text import Text
-from pipelex.system.telemetry.telemetry_config import TelemetryMode
from pipelex.tools.misc.file_utils import path_exists
from pipelex.tools.misc.string_utils import snake_to_capitalize_first_letter
from pipelex.tools.misc.toml_utils import load_toml_from_path
-from pipelex.types import StrEnum
def get_backend_options_from_toml(template_path: str, existing_path: str | None = None) -> list[tuple[str, str]]:
@@ -100,15 +92,6 @@ def get_currently_enabled_backends(backends_toml_path: str, backend_options: lis
return sorted(currently_enabled)
-class InitFocus(StrEnum):
- """Focus options for initialization."""
-
- ALL = "all"
- CONFIG = "config"
- INFERENCE = "inference"
- TELEMETRY = "telemetry"
-
-
def build_backend_selection_panel(backend_options: list[tuple[str, str]], currently_enabled: list[int] | None = None) -> Panel:
"""Create a Rich Panel for backend selection with options table.
@@ -248,190 +231,3 @@ def display_selected_backends(console: Console, selected_indices: list[int], bac
console.print(f"\n[green]✓[/green] Configured {len(selected_names)} backend(s):")
for name in selected_names:
console.print(f" • {name}")
-
-
-def build_telemetry_selection_panel() -> Panel:
- """Create a Rich Panel for telemetry mode selection.
-
- Returns:
- A Panel containing the telemetry selection interface.
- """
- table = Table(show_header=False, box=None, padding=(0, 2))
- table.add_column(style="bold cyan", justify="right")
- table.add_column(style="bold")
- table.add_column()
-
- table.add_row("[1]", TelemetryMode.OFF, "No telemetry data collected")
- table.add_row("[2]", TelemetryMode.ANONYMOUS, "Anonymous usage data only")
- table.add_row("[3]", TelemetryMode.IDENTIFIED, "Usage data with user identification")
- table.add_row("[Q]", "[dim]quit[/dim]", "[dim]Exit without configuring[/dim]")
-
- description = Text(
- "Pipelex can collect anonymous usage data to help improve the product.",
- style="dim",
- )
-
- return Panel(
- Group(description, Text(""), table),
- title="[bold yellow]Telemetry Configuration[/bold yellow]",
- border_style="yellow",
- padding=(1, 2),
- )
-
-
-def prompt_telemetry_mode(console: Console) -> TelemetryMode:
- """Prompt user to select telemetry mode with validation.
-
- Args:
- console: Rich Console instance for user interaction.
-
- Returns:
- Selected TelemetryMode.
-
- Raises:
- typer.Exit: If user chooses to quit.
- """
- # Map choice to telemetry mode
- mode_map: dict[str, TelemetryMode] = {
- "1": TelemetryMode.OFF,
- "2": TelemetryMode.ANONYMOUS,
- "3": TelemetryMode.IDENTIFIED,
- "off": TelemetryMode.OFF,
- "anonymous": TelemetryMode.ANONYMOUS,
- "identified": TelemetryMode.IDENTIFIED,
- }
-
- # Loop until valid input
- telemetry_mode: TelemetryMode | None = None
- while telemetry_mode is None:
- choice_str = Prompt.ask("[bold]Enter your choice[/bold]", console=console)
- choice_input = choice_str.lower().strip()
-
- # Handle quit option
- if choice_input in ("q", "quit"):
- console.print("\n[yellow]Exiting without configuring telemetry.[/yellow]")
- raise typer.Exit(code=0)
-
- if choice_input in mode_map:
- telemetry_mode = mode_map[choice_input]
- else:
- console.print(
- f"[red]Invalid choice: '{choice_str}'.[/red] "
- "Please enter [cyan]1[/cyan], [cyan]2[/cyan], [cyan]3[/cyan], or [cyan]q[/cyan] to quit.\n"
- )
-
- return telemetry_mode
-
-
-def display_already_configured_message(focus: InitFocus, console: Console, config_path: str) -> bool:
- """Display 'already configured' message and ask if user wants to reconfigure.
-
- Args:
- focus: The initialization focus area.
- console: Rich Console instance for output.
- config_path: Path to the configuration file.
-
- Returns:
- True if user wants to reconfigure, False otherwise.
- """
- # Mapping of focus to (subject, action_verb)
- focus_messages = {
- InitFocus.INFERENCE: ("Inference backends", "inference backends"),
- InitFocus.TELEMETRY: ("Telemetry preferences", "telemetry preferences"),
- InitFocus.CONFIG: ("Configuration files", "configuration"),
- }
-
- if focus == InitFocus.ALL:
- console.print()
- console.print("[green]✓[/green] Pipelex is already fully initialized!")
- console.print()
- console.print("[dim]Configuration files are in place:[/dim] [cyan].pipelex/[/cyan]")
- console.print("[dim]Telemetry preferences are configured[/dim]")
- console.print()
- console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
- console.print(" [cyan]pipelex init --reset[/cyan]")
- console.print()
- return False
-
- if focus == InitFocus.CONFIG:
- console.print()
- console.print("[green]✓[/green] Configuration files are already in place!")
- console.print()
- console.print("[dim]Configuration directory:[/dim] [cyan].pipelex/[/cyan]")
- console.print()
- console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
- console.print(f" [cyan]pipelex init {focus} --reset[/cyan]")
- console.print()
- return False
-
- if focus in focus_messages:
- subject, action_verb = focus_messages[focus]
- console.print()
- console.print(f"[green]✓[/green] {subject} are already configured!")
- console.print()
- console.print(f"[dim]Configuration file:[/dim] [cyan]{config_path}[/cyan]")
- console.print()
-
- return Confirm.ask(f"[bold]Would you like to reconfigure {action_verb}?[/bold]", default=False)
-
- return False
-
-
-def build_initialization_panel(needs_config: bool, needs_inference: bool, needs_telemetry: bool, reset: bool) -> Panel:
- """Build the initialization confirmation panel.
-
- Args:
- needs_config: Whether config initialization is needed.
- needs_inference: Whether inference setup is needed.
- needs_telemetry: Whether telemetry setup is needed.
- reset: Whether this is a reset operation.
-
- Returns:
- A Panel containing the initialization confirmation message.
- """
- # Build message based on what's being initialized
- message_parts: list[str] = []
- if reset:
- if needs_config:
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")
- if needs_inference:
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] inference backends")
- if needs_telemetry:
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] telemetry preferences")
- else:
- if needs_config:
- message_parts.append("• Create required configuration files in [cyan].pipelex/[/cyan]")
- if needs_inference:
- message_parts.append("• Ask you to choose your inference backends")
- if needs_telemetry:
- message_parts.append("• Ask you to choose your telemetry preferences")
-
- # Determine title based on what's being initialized
- num_items = sum([needs_config, needs_inference, needs_telemetry])
- if reset:
- if num_items > 1:
- title_text = "[bold yellow]Resetting Configuration[/bold yellow]"
- elif needs_config:
- title_text = "[bold yellow]Resetting Configuration Files[/bold yellow]"
- elif needs_inference:
- title_text = "[bold yellow]Resetting Inference Backends[/bold yellow]"
- else:
- title_text = "[bold yellow]Resetting Telemetry[/bold yellow]"
- elif num_items > 1:
- title_text = "[bold cyan]Pipelex Initialization[/bold cyan]"
- elif needs_config:
- title_text = "[bold cyan]Configuration Setup[/bold cyan]"
- elif needs_inference:
- title_text = "[bold cyan]Inference Backend Setup[/bold cyan]"
- else:
- title_text = "[bold cyan]Telemetry Setup[/bold cyan]"
-
- message = "\n".join(message_parts)
- border_color = "yellow" if reset else "cyan"
-
- return Panel(
- message,
- title=title_text,
- border_style=border_color,
- padding=(1, 2),
- )
diff --git a/pipelex/cli/commands/init/ui/general_ui.py b/pipelex/cli/commands/init/ui/general_ui.py
new file mode 100644
index 000000000..eedc5c474
--- /dev/null
+++ b/pipelex/cli/commands/init/ui/general_ui.py
@@ -0,0 +1,134 @@
+"""General UI components for the init command."""
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.prompt import Confirm
+
+from pipelex.cli.commands.init.ui.types import InitFocus
+
+
+def display_already_configured_message(focus: InitFocus, console: Console, config_path: str) -> bool:
+ """Display 'already configured' message and ask if user wants to reconfigure.
+
+ Args:
+ focus: The initialization focus area.
+ console: Rich Console instance for output.
+ config_path: Path to the configuration file.
+
+ Returns:
+ True if user wants to reconfigure, False otherwise.
+ """
+ # Mapping of focus to (subject, action_verb)
+ focus_messages = {
+ InitFocus.INFERENCE: ("Inference backends", "inference backends"),
+ InitFocus.ROUTING: ("Routing profile", "routing profile"),
+ InitFocus.TELEMETRY: ("Telemetry preferences", "telemetry preferences"),
+ InitFocus.CONFIG: ("Configuration files", "configuration"),
+ }
+
+ if focus == InitFocus.ALL:
+ console.print()
+ console.print("[green]✓[/green] Pipelex is already fully initialized!")
+ console.print()
+ console.print("[dim]Configuration files are in place:[/dim] [cyan].pipelex/[/cyan]")
+ console.print("[dim]Telemetry preferences are configured[/dim]")
+ console.print()
+ console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
+ console.print(" [cyan]pipelex init --reset[/cyan]")
+ console.print()
+ return False
+
+ if focus == InitFocus.CONFIG:
+ console.print()
+ console.print("[green]✓[/green] Configuration files are already in place!")
+ console.print()
+ console.print("[dim]Configuration directory:[/dim] [cyan].pipelex/[/cyan]")
+ console.print()
+ console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")
+ console.print(f" [cyan]pipelex init {focus} --reset[/cyan]")
+ console.print()
+ return False
+
+ if focus in focus_messages:
+ subject, action_verb = focus_messages[focus]
+ console.print()
+ if focus == InitFocus.ROUTING:
+ console.print(f"[green]✓[/green] {subject} is already configured!")
+ else:
+ console.print(f"[green]✓[/green] {subject} are already configured!")
+ console.print()
+ console.print(f"[dim]Configuration file:[/dim] [cyan]{config_path}[/cyan]")
+ console.print()
+
+ return Confirm.ask(f"[bold]Would you like to reconfigure {action_verb}?[/bold]", default=False)
+
+ return False
+
+
+def build_initialization_panel(needs_config: bool, needs_inference: bool, needs_routing: bool, needs_telemetry: bool, reset: bool) -> Panel:
+ """Build the initialization confirmation panel.
+
+ Args:
+ needs_config: Whether config initialization is needed.
+ needs_inference: Whether inference setup is needed.
+ needs_routing: Whether routing setup is needed.
+ needs_telemetry: Whether telemetry setup is needed.
+ reset: Whether this is a reset operation.
+
+ Returns:
+ A Panel containing the initialization confirmation message.
+ """
+ # Build message based on what's being initialized
+ message_parts: list[str] = []
+ if reset:
+ if needs_config:
+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")
+ if needs_inference:
+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] inference backends")
+ if needs_routing:
+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] routing profile")
+ if needs_telemetry:
+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] telemetry preferences")
+ else:
+ if needs_config:
+ message_parts.append("• Create required configuration files in [cyan].pipelex/[/cyan]")
+ if needs_inference:
+ message_parts.append("• Ask you to choose your inference backends")
+ if needs_routing:
+ message_parts.append("• Ask you to configure your routing profile")
+ if needs_telemetry:
+ message_parts.append("• Ask you to choose your telemetry preferences")
+
+ # Determine title based on what's being initialized
+ num_items = sum([needs_config, needs_inference, needs_routing, needs_telemetry])
+ if reset:
+ if num_items > 1:
+ title_text = "[bold yellow]Resetting Configuration[/bold yellow]"
+ elif needs_config:
+ title_text = "[bold yellow]Resetting Configuration Files[/bold yellow]"
+ elif needs_inference:
+ title_text = "[bold yellow]Resetting Inference Backends[/bold yellow]"
+ elif needs_routing:
+ title_text = "[bold yellow]Resetting Routing Profile[/bold yellow]"
+ else:
+ title_text = "[bold yellow]Resetting Telemetry[/bold yellow]"
+ elif num_items > 1:
+ title_text = "[bold cyan]Pipelex Initialization[/bold cyan]"
+ elif needs_config:
+ title_text = "[bold cyan]Configuration Setup[/bold cyan]"
+ elif needs_inference:
+ title_text = "[bold cyan]Inference Backend Setup[/bold cyan]"
+ elif needs_routing:
+ title_text = "[bold cyan]Routing Profile Setup[/bold cyan]"
+ else:
+ title_text = "[bold cyan]Telemetry Setup[/bold cyan]"
+
+ message = "\n".join(message_parts)
+ border_color = "yellow" if reset else "cyan"
+
+ return Panel(
+ message,
+ title=title_text,
+ border_style=border_color,
+ padding=(1, 2),
+ )
diff --git a/pipelex/cli/commands/init/ui/routing_ui.py b/pipelex/cli/commands/init/ui/routing_ui.py
new file mode 100644
index 000000000..082f29ddd
--- /dev/null
+++ b/pipelex/cli/commands/init/ui/routing_ui.py
@@ -0,0 +1,207 @@
+"""UI components for routing profile configuration in the init command."""
+
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
+from rich.text import Text
+
+from pipelex.tools.misc.string_utils import snake_to_capitalize_first_letter
+
+
+def build_primary_backend_panel(backend_keys: list[str], backend_options: list[tuple[str, str]]) -> Panel:
+ """Build a panel for selecting a primary backend from enabled backends.
+
+ Args:
+ backend_keys: List of enabled backend keys.
+ backend_options: List of all backend options to get display names.
+
+ Returns:
+ A Panel containing the primary backend selection interface.
+ """
+ # Create a mapping of backend_key to display_name
+ backend_key_to_name = dict(backend_options)
+
+ table = Table(show_header=False, box=None, padding=(0, 2))
+ table.add_column(style="bold cyan", justify="right", width=4)
+ table.add_column(style="bold", width=30)
+
+ for idx, backend_key in enumerate(backend_keys, start=1):
+ display_name = backend_key_to_name.get(backend_key, snake_to_capitalize_first_letter(backend_key))
+ table.add_row(f"[{idx}]", display_name)
+
+ description = Text(
+ "Multiple backends are enabled. Select which backend should be your primary/default.\n"
+ "The primary backend will be used for models that don't have an exact match or pattern match in your routes.",
+ style="dim",
+ )
+
+ return Panel(
+ Group(description, Text(""), table),
+ title="[bold yellow]Primary Backend Selection[/bold yellow]",
+ border_style="yellow",
+ padding=(1, 2),
+ )
+
+
+def prompt_primary_backend(console: Console, backend_keys: list[str]) -> str:
+ """Prompt user to select a primary backend from enabled backends.
+
+ Args:
+ console: Rich Console instance for user interaction.
+ backend_keys: List of enabled backend keys.
+
+ Returns:
+ The selected backend key.
+ """
+ # Default to first backend
+ default_str = "1"
+
+ selected_backend: str | None = None
+ while selected_backend is None:
+ choice_str = Prompt.ask("[bold]Enter your choice[/bold]", default=default_str, console=console)
+ choice_input = choice_str.strip()
+
+ # Parse input - handle empty (use default)
+ if not choice_input:
+ selected_backend = backend_keys[0]
+ break
+
+ try:
+ # Parse as 1-based index from user input
+ user_index = int(choice_input)
+
+ # Validate index is in range (1-based)
+ if user_index < 1 or user_index > len(backend_keys):
+ max_idx = len(backend_keys)
+ console.print(f"[red]Invalid choice: {user_index}.[/red] Please enter a number between 1 and {max_idx}.\n")
+ continue
+
+ # Convert to 0-based index and get backend key
+ selected_backend = backend_keys[user_index - 1]
+ break
+
+ except ValueError:
+ console.print(f"[red]Invalid input: '{choice_str}'.[/red] Please enter a number.\n")
+
+ return selected_backend
+
+
+def build_fallback_order_panel(remaining_backends: list[str], backend_options: list[tuple[str, str]]) -> Panel:
+ """Build a panel explaining fallback order configuration.
+
+ Args:
+ remaining_backends: List of backend keys (excluding primary).
+ backend_options: List of all backend options to get display names.
+
+ Returns:
+ A Panel containing the fallback order explanation and current order.
+ """
+ # Create a mapping of backend_key to display_name
+ backend_key_to_name = dict(backend_options)
+
+ table = Table(show_header=False, box=None, padding=(0, 2))
+ table.add_column(style="bold cyan", justify="right", width=4)
+ table.add_column(style="bold", width=30)
+
+ for idx, backend_key in enumerate(remaining_backends, start=1):
+ display_name = backend_key_to_name.get(backend_key, snake_to_capitalize_first_letter(backend_key))
+ table.add_row(f"[{idx}]", display_name)
+
+ description = Text(
+ "Fallback order determines which backends to try when a model has no exact match or pattern match\n"
+ "and is not available in your default backend.\nCurrent order:",
+ style="dim",
+ )
+
+ return Panel(
+ Group(description, Text(""), table),
+ title="[bold yellow]Fallback Backend Order[/bold yellow]",
+ border_style="yellow",
+ padding=(1, 2),
+ )
+
+
+def prompt_fallback_order(console: Console, remaining_backends: list[str], backend_options: list[tuple[str, str]]) -> list[str]:
+ """Prompt user to order the remaining backends for fallback.
+
+ Args:
+ console: Rich Console instance for user interaction.
+ remaining_backends: List of backend keys (excluding primary).
+ backend_options: List of all backend options to get display names.
+
+ Returns:
+ Ordered list of backend keys for fallback.
+ """
+ backend_key_to_name = dict(backend_options)
+
+ console.print()
+ console.print("[dim]Enter the order of backends (space or comma separated indices).[/dim]")
+ console.print("[dim]For example: '2 1 3' or '2,1,3' to reorder them.[/dim]")
+ console.print()
+
+ fallback_order: list[str] | None = None
+ while fallback_order is None:
+ # Show current order as default
+ default_order = " ".join(str(i + 1) for i in range(len(remaining_backends)))
+ choice_str = Prompt.ask("[bold]Enter order[/bold]", default=default_order, console=console)
+ choice_input = choice_str.strip()
+
+ # Handle empty input (use default order)
+ if not choice_input:
+ fallback_order = remaining_backends.copy()
+ break
+
+ # Split by comma or space
+ parts = choice_input.replace(",", " ").split()
+
+ try:
+ # Parse as 1-based indices from user input
+ user_indices = [int(part.strip()) for part in parts if part.strip()]
+
+ # Validate: must have exactly the same number of indices
+ if len(user_indices) != len(remaining_backends):
+ console.print(f"[red]Error: You must specify all {len(remaining_backends)} backends.[/red] You provided {len(user_indices)}.\n")
+ continue
+
+ # Validate: all indices are in range (1-based)
+ invalid_indices = [i for i in user_indices if i < 1 or i > len(remaining_backends)]
+ if invalid_indices:
+ max_idx = len(remaining_backends)
+ console.print(f"[red]Invalid choice(s): {invalid_indices}.[/red] Please enter numbers between 1 and {max_idx}.\n")
+ continue
+
+ # Validate: no duplicates
+ if len(user_indices) != len(set(user_indices)):
+ console.print("[red]Error: Duplicate indices not allowed.[/red] Each backend must appear exactly once.\n")
+ continue
+
+ # Convert to 0-based indices and reorder backends
+ zero_based_indices = [i - 1 for i in user_indices]
+ fallback_order = [remaining_backends[i] for i in zero_based_indices]
+ break
+
+ except ValueError:
+ console.print(f"[red]Invalid input: '{choice_str}'.[/red] Please enter numbers separated by commas or spaces.\n")
+
+ # Display the final order
+ console.print("\n[green]✓[/green] Fallback order set to:")
+ for idx, backend_key in enumerate(fallback_order, start=1):
+ display_name = backend_key_to_name.get(backend_key, snake_to_capitalize_first_letter(backend_key))
+ console.print(f" {idx}. {display_name}")
+
+ return fallback_order
+
+
+def display_routing_profile_result(console: Console, profile_name: str, created: bool = False) -> None:
+ """Display confirmation of routing profile configuration.
+
+ Args:
+ console: Rich Console instance for output.
+ profile_name: Name of the routing profile that was set.
+ created: Whether the profile was newly created.
+ """
+ if created:
+ console.print(f"\n[green]✓[/green] Created and set routing profile to: [bold cyan]{profile_name}[/bold cyan]")
+ else:
+ console.print(f"\n[green]✓[/green] Routing profile set to: [bold cyan]{profile_name}[/bold cyan]")
diff --git a/pipelex/cli/commands/init/ui/telemetry_ui.py b/pipelex/cli/commands/init/ui/telemetry_ui.py
new file mode 100644
index 000000000..0a338d74d
--- /dev/null
+++ b/pipelex/cli/commands/init/ui/telemetry_ui.py
@@ -0,0 +1,83 @@
+"""UI components for telemetry configuration in the init command."""
+
+import typer
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
+from rich.text import Text
+
+from pipelex.system.telemetry.telemetry_config import TelemetryMode
+
+
+def build_telemetry_selection_panel() -> Panel:
+ """Create a Rich Panel for telemetry mode selection.
+
+ Returns:
+ A Panel containing the telemetry selection interface.
+ """
+ table = Table(show_header=False, box=None, padding=(0, 2))
+ table.add_column(style="bold cyan", justify="right")
+ table.add_column(style="bold")
+ table.add_column()
+
+ table.add_row("[1]", TelemetryMode.OFF, "No telemetry data collected")
+ table.add_row("[2]", TelemetryMode.ANONYMOUS, "Anonymous usage data only")
+ table.add_row("[3]", TelemetryMode.IDENTIFIED, "Usage data with user identification")
+ table.add_row("[Q]", "[dim]quit[/dim]", "[dim]Exit without configuring[/dim]")
+
+ description = Text(
+ "Pipelex can collect anonymous usage data to help improve the product.",
+ style="dim",
+ )
+
+ return Panel(
+ Group(description, Text(""), table),
+ title="[bold yellow]Telemetry Configuration[/bold yellow]",
+ border_style="yellow",
+ padding=(1, 2),
+ )
+
+
+def prompt_telemetry_mode(console: Console) -> TelemetryMode:
+ """Prompt user to select telemetry mode with validation.
+
+ Args:
+ console: Rich Console instance for user interaction.
+
+ Returns:
+ Selected TelemetryMode.
+
+ Raises:
+ typer.Exit: If user chooses to quit.
+ """
+ # Map choice to telemetry mode
+ mode_map: dict[str, TelemetryMode] = {
+ "1": TelemetryMode.OFF,
+ "2": TelemetryMode.ANONYMOUS,
+ "3": TelemetryMode.IDENTIFIED,
+ "off": TelemetryMode.OFF,
+ "anonymous": TelemetryMode.ANONYMOUS,
+ "identified": TelemetryMode.IDENTIFIED,
+ }
+
+ # Loop until valid input
+ telemetry_mode: TelemetryMode | None = None
+ while telemetry_mode is None:
+ choice_str = Prompt.ask("[bold]Enter your choice[/bold]", console=console)
+ choice_input = choice_str.lower().strip()
+
+ # Handle quit option
+ if choice_input in ("q", "quit"):
+ console.print("\n[yellow]Exiting without configuring telemetry.[/yellow]")
+ raise typer.Exit(code=0)
+
+ if choice_input in mode_map:
+ telemetry_mode = mode_map[choice_input]
+ else:
+ console.print(
+ f"[red]Invalid choice: '{choice_str}'.[/red] "
+ "Please enter [cyan]1[/cyan], [cyan]2[/cyan], [cyan]3[/cyan], or [cyan]q[/cyan] to quit.\n"
+ )
+
+ return telemetry_mode
diff --git a/pipelex/cli/commands/init/ui/types.py b/pipelex/cli/commands/init/ui/types.py
new file mode 100644
index 000000000..274ce2115
--- /dev/null
+++ b/pipelex/cli/commands/init/ui/types.py
@@ -0,0 +1,13 @@
+"""Types for the init command UI."""
+
+from pipelex.types import StrEnum
+
+
+class InitFocus(StrEnum):
+ """Focus options for initialization."""
+
+ ALL = "all"
+ CONFIG = "config"
+ INFERENCE = "inference"
+ ROUTING = "routing"
+ TELEMETRY = "telemetry"
diff --git a/pipelex/cli/commands/run_cmd.py b/pipelex/cli/commands/run_cmd.py
index 8572f9050..f3e804054 100644
--- a/pipelex/cli/commands/run_cmd.py
+++ b/pipelex/cli/commands/run_cmd.py
@@ -6,13 +6,28 @@ from typing import Annotated
import click
import typer
-from posthog import new_context, tag
+from posthog import tag
from rich.console import Console
from pipelex import log, pretty_print_md
from pipelex.builder.builder import load_and_validate_bundle
from pipelex.builder.builder_errors import PipelexBundleError
-from pipelex.exceptions import PipeInputError, PipelineExecutionError
+from pipelex.cli.error_handlers import (
+ ErrorContext,
+ handle_model_availability_error,
+ handle_model_choice_error,
+ handle_model_deck_preset_error,
+ handle_validation_error,
+)
+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
+from pipelex.exceptions import (
+ LibraryLoadingError,
+ PipeInputError,
+ PipelineExecutionError,
+ PipeOperatorModelAvailabilityError,
+ PipeOperatorModelChoiceError,
+)
+from pipelex.hub import get_telemetry_manager
from pipelex.pipelex import Pipelex
from pipelex.pipeline.execute import execute_pipeline
from pipelex.system.runtime import IntegrationMode
@@ -111,8 +126,6 @@ def run_cmd(
raise typer.Exit(1)
async def run_pipeline(pipe_code: str | None = None, bundle_path: str | None = None):
- # Initialize Pipelex
- Pipelex.make(integration_mode=IntegrationMode.CLI)
source_description: str
if bundle_path:
try:
@@ -141,63 +154,79 @@ def run_cmd(
typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)
raise typer.Exit(1)
- try:
- # Load inputs if provided
- pipeline_inputs = None
- if inputs:
- if inputs.startswith("{"):
- pipeline_inputs = json.loads(inputs)
- else:
- try:
- pipeline_inputs = load_json_dict_from_path(inputs)
- typer.echo(f"Loaded inputs from: {inputs}")
- except FileNotFoundError as file_not_found_exc:
- typer.secho(f"Failed to load input file '{inputs}': file not found", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from file_not_found_exc
- except JsonTypeError as json_type_error_exc:
- typer.secho(f"Failed to parse input file '{inputs}': must be a valid JSON dictionary", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from json_type_error_exc
-
- # Execute pipeline
- typer.secho(f"\n🚀 Executing {source_description}...\n", fg=typer.colors.GREEN, bold=True)
-
- try:
- pipe_output = await execute_pipeline(
- pipe_code=pipe_code,
- inputs=pipeline_inputs,
- )
- except PipelineExecutionError as exc:
- typer.secho(f"Failed to execute pipeline: {exc}", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from exc
-
- # Pretty print main_stuff unless disabled
- if not no_pretty_print:
- typer.echo("")
- pretty_print_md(content=pipe_output.main_stuff.content.rendered_markdown(), title=f"Main output of '{pipe_code}'")
- typer.echo("")
-
- # Save working memory to JSON unless disabled
- if not no_output:
- output_path = output or get_incremental_file_path(
- base_path="results",
- base_name=f"run_{pipe_code}",
- extension="json",
- )
- working_memory_dict = pipe_output.working_memory.smart_dump()
- save_as_json_to_path(object_to_save=working_memory_dict, path=output_path)
- typer.secho(f"✅ Working memory saved to: {output_path}", fg=typer.colors.GREEN)
+ # Load inputs if provided
+ pipeline_inputs = None
+ if inputs:
+ if inputs.startswith("{"):
+ pipeline_inputs = json.loads(inputs)
+ else:
+ try:
+ pipeline_inputs = load_json_dict_from_path(inputs)
+ typer.echo(f"Loaded inputs from: {inputs}")
+ except FileNotFoundError as file_not_found_exc:
+ typer.secho(f"Failed to load input file '{inputs}': file not found", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from file_not_found_exc
+ except JsonTypeError as json_type_error_exc:
+ typer.secho(f"Failed to parse input file '{inputs}': must be a valid JSON dictionary", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from json_type_error_exc
+
+ # Execute pipeline
+ typer.secho(f"\n🚀 Executing {source_description}...\n", fg=typer.colors.GREEN, bold=True)
- typer.secho("✅ Pipeline execution completed successfully", fg=typer.colors.GREEN)
-
- except Exception as exc:
- log.error(f"Error executing pipeline: {exc}")
- console = Console(stderr=True)
- console.print("\n[bold red]Failed to execute pipeline[/bold red]\n")
- console.print_exception(show_locals=True)
+ try:
+ pipe_output = await execute_pipeline(
+ pipe_code=pipe_code,
+ inputs=pipeline_inputs,
+ )
+ except PipelineExecutionError as exc:
+ typer.secho(f"Failed to execute pipeline: {exc}", fg=typer.colors.RED, err=True)
raise typer.Exit(1) from exc
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
- tag(name=EventProperty.CLI_COMMAND, value=COMMAND)
- asyncio.run(run_pipeline(pipe_code=pipe_code, bundle_path=bundle_path))
+ # Pretty print main_stuff unless disabled
+ if not no_pretty_print:
+ typer.echo("")
+ pretty_print_md(content=pipe_output.main_stuff.content.rendered_markdown(), title=f"Main output of '{pipe_code}'")
+ typer.echo("")
+
+ # Save working memory to JSON unless disabled
+ if not no_output:
+ output_path = output or get_incremental_file_path(
+ base_path="results",
+ base_name=f"run_{pipe_code}",
+ extension="json",
+ )
+ working_memory_dict = pipe_output.working_memory.smart_dump()
+ save_as_json_to_path(object_to_save=working_memory_dict, path=output_path)
+ typer.secho(f"✅ Working memory saved to: {output_path}", fg=typer.colors.GREEN)
+
+ typer.secho("✅ Pipeline execution completed successfully", fg=typer.colors.GREEN)
+
+ # Initialize Pipelex BEFORE telemetry context to ensure proper setup
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_PIPE_RUN)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_PIPE_RUN)
+
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
+ tag(name=EventProperty.CLI_COMMAND, value=COMMAND)
+ asyncio.run(run_pipeline(pipe_code=pipe_code, bundle_path=bundle_path))
+
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.PIPE_RUN)
+
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.PIPE_RUN)
+
+ except Exception as exc:
+ log.error(f"Error executing pipeline: {exc}")
+ console = Console(stderr=True)
+ console.print("\n[bold red]Failed to execute pipeline[/bold red]\n")
+ console.print_exception(show_locals=True)
+ raise typer.Exit(1) from exc
+ finally:
+ pipelex_instance.teardown()
diff --git a/pipelex/cli/commands/show_cmd.py b/pipelex/cli/commands/show_cmd.py
index 931f50a85..63a45bb2c 100644
--- a/pipelex/cli/commands/show_cmd.py
+++ b/pipelex/cli/commands/show_cmd.py
@@ -4,16 +4,23 @@ import asyncio
from typing import TYPE_CHECKING, Annotated, cast
import typer
-from posthog import new_context, tag
+from posthog import tag
from rich import box
from rich.console import Console
from rich.table import Table
from pipelex import pretty_print
+from pipelex.cli.error_handlers import (
+ ErrorContext,
+ handle_model_deck_preset_error,
+ handle_validation_error,
+)
+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
from pipelex.cogt.model_backends.backend_library import InferenceBackendLibrary
from pipelex.cogt.model_backends.model_lists import ModelLister
-from pipelex.exceptions import PipelexCLIError, PipelexConfigError
-from pipelex.hub import get_models_manager, get_pipe_library, get_required_pipe, get_telemetry_manager
+from pipelex.config import ConfigPaths
+from pipelex.exceptions import LibraryLoadingError, PipelexCLIError, PipelexConfigError
+from pipelex.hub import get_models_manager, get_pipe_library, get_required_pipe, get_secrets_provider, get_telemetry_manager
from pipelex.pipelex import Pipelex
from pipelex.system.configuration.config_loader import config_manager
from pipelex.system.runtime import IntegrationMode
@@ -60,16 +67,22 @@ def do_show_pipe(pipe_code: str) -> None:
def do_show_backends(show_all: bool = False) -> None:
"""Display all backends and the active routing profile."""
try:
+ secrets_provider = get_secrets_provider()
models_manager = cast("ModelManager", get_models_manager())
# Load backends with or without disabled ones based on show_all flag
if show_all:
backend_library = InferenceBackendLibrary()
- backend_library.load(include_disabled=True)
+ backend_library.load(
+ secrets_provider=secrets_provider,
+ backends_library_path=ConfigPaths.BACKENDS_FILE_PATH,
+ backends_dir_path=ConfigPaths.BACKENDS_DIR_PATH,
+ include_disabled=True,
+ )
else:
backend_library = models_manager.inference_backend_library
- routing_profile_library = models_manager.routing_profile_library
+ routing_profile = models_manager.routing_profile
except Exception as exc:
msg = f"Error accessing backend or routing configuration: {exc}"
raise PipelexCLIError(msg) from exc
@@ -115,17 +128,15 @@ def do_show_backends(show_all: bool = False) -> None:
# Display routing profile information
try:
- active_profile = routing_profile_library.active_profile
-
- console.print(f"[bold cyan]Active Routing Profile:[/bold cyan] [green]{active_profile.name}[/green]")
- if active_profile.description:
- console.print(f"[dim]{active_profile.description}[/dim]")
+ console.print(f"[bold cyan]Active Routing Profile:[/bold cyan] [green]{routing_profile.name}[/green]")
+ if routing_profile.description:
+ console.print(f"[dim]{routing_profile.description}[/dim]")
- if active_profile.default:
- console.print(f"[bold]Default Backend:[/bold] [cyan]{active_profile.default}[/cyan]")
+ if routing_profile.default:
+ console.print(f"[bold]Default Backend:[/bold] [cyan]{routing_profile.default}[/cyan]")
# Display routing rules
- if active_profile.routes:
+ if routing_profile.routes:
console.print("\n[bold]Routing Rules:[/bold]")
routes_table = Table(
show_header=True,
@@ -137,7 +148,7 @@ def do_show_backends(show_all: bool = False) -> None:
routes_table.add_column("→", style="dim", justify="center")
routes_table.add_column("Target Backend", style="cyan")
- for pattern, target_backend in sorted(active_profile.routes.items()):
+ for pattern, target_backend in sorted(routing_profile.routes.items()):
routes_table.add_row(pattern, "→", target_backend)
console.print(routes_table)
@@ -180,9 +191,14 @@ def list_pipes_cmd() -> None:
This includes pipes from your project's .plx files and any
pipes from imported packages.
"""
- Pipelex.make(integration_mode=IntegrationMode.CLI)
+ try:
+ Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPES)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPES)
- with new_context():
+ with get_telemetry_manager().telemetry_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPES}")
@@ -200,9 +216,14 @@ def show_pipe_cmd(
Example:
pipelex show pipe hello_world
"""
- Pipelex.make(integration_mode=IntegrationMode.CLI)
+ try:
+ Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPE)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPE)
- with new_context():
+ with get_telemetry_manager().telemetry_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPE}")
@@ -227,18 +248,27 @@ def show_models_cmd(
pipelex show models openai
pipelex show models anthropic --flat
"""
- Pipelex.make(integration_mode=IntegrationMode.CLI)
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_MODELS}")
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_MODELS)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_MODELS)
- asyncio.run(
- ModelLister.list_models(
- backend_name=backend_name,
- flat=flat,
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_MODELS}")
+
+ asyncio.run(
+ ModelLister.list_models(
+ backend_name=backend_name,
+ flat=flat,
+ )
)
- )
+ finally:
+ pipelex_instance.teardown()
@show_app.command("backends", help="Display backend configurations and active routing profile")
@@ -253,8 +283,14 @@ def show_backends_cmd(
pipelex show backends
pipelex show backends --all
"""
- Pipelex.make(integration_mode=IntegrationMode.CLI)
- with new_context():
+ try:
+ Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_BACKENDS)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_BACKENDS)
+
+ with get_telemetry_manager().telemetry_context():
tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_BACKENDS}")
diff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.py
index 756635d7f..92c4d4b6e 100644
--- a/pipelex/cli/commands/validate_cmd.py
+++ b/pipelex/cli/commands/validate_cmd.py
@@ -6,16 +6,23 @@ from typing import TYPE_CHECKING, Annotated
import click
import typer
-from posthog import new_context, tag
+from posthog import tag
from rich.console import Console
-from rich.syntax import Syntax
from rich.traceback import Traceback
-from pipelex import log, pretty_print
+from pipelex import log
from pipelex.builder.builder_errors import PipelexBundleError
from pipelex.builder.builder_validation import validate_dry_run_bundle_blueprint
+from pipelex.cli.error_handlers import (
+ ErrorContext,
+ handle_model_availability_error,
+ handle_model_choice_error,
+ handle_model_deck_preset_error,
+ handle_validation_error,
+)
+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.exceptions import LibraryLoadingError, PipeInputError
+from pipelex.exceptions import LibraryLoadingError, PipeInputError, PipeOperatorModelAvailabilityError, PipeOperatorModelChoiceError
from pipelex.hub import get_library_manager, get_pipes, get_required_pipe, get_telemetry_manager
from pipelex.pipe_run.dry_run import dry_run_pipe, dry_run_pipes
from pipelex.pipelex import Pipelex
@@ -25,7 +32,6 @@ from pipelex.tools.misc.package_utils import get_package_version
if TYPE_CHECKING:
from pipelex.core.pipes.pipe_abstract import PipeAbstract
- from pipelex.core.validation_errors import ValidationErrorDetailsProtocol
console = Console()
@@ -34,17 +40,30 @@ COMMAND = "validate"
def do_validate_all_libraries_and_dry_run() -> None:
"""Validate libraries and dry-run all pipes."""
- pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} all")
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except LibraryLoadingError as library_loading_error:
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION)
- pipelex_instance.validate_libraries()
- pipes = get_pipes()
- get_telemetry_manager().track_event(EventName.PIPE_DRY_RUN, properties={EventProperty.NB_PIPES: len(pipes)})
- asyncio.run(dry_run_pipes(pipes=pipes, raise_on_failure=True))
- log.info("Setup sequence passed OK, config and pipelines are validated.")
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} all")
+
+ pipelex_instance.validate_libraries()
+ pipes = get_pipes()
+ get_telemetry_manager().track_event(EventName.PIPE_DRY_RUN, properties={EventProperty.NB_PIPES: len(pipes)})
+ asyncio.run(dry_run_pipes(pipes=pipes, raise_on_failure=True))
+ log.info("Setup sequence passed OK, config and pipelines are validated.")
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.VALIDATION)
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.VALIDATION)
+ finally:
+ pipelex_instance.teardown()
def validate_cmd(
@@ -152,9 +171,7 @@ def validate_cmd(
raise typer.Exit(1) from exc
except PipelexBundleError as bundle_error:
console.print(Traceback())
- typer.secho(f"\n❌ Failed to validate bundle '{bundle_path}':", fg=typer.colors.RED, err=True)
- present_validation_error(details_provider=bundle_error)
- raise typer.Exit(1) from bundle_error
+ handle_validation_error(exc=bundle_error, context=ErrorContext.VALIDATION)
except PipeInputError as exc:
console.print(Traceback())
typer.secho(f"\n❌ Failed to validate bundle '{bundle_path}':", fg=typer.colors.RED, err=True)
@@ -176,58 +193,27 @@ def validate_cmd(
raise typer.Exit(1)
# Initialize Pipelex
+ pipelex_instance: Pipelex
try:
pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
except LibraryLoadingError as library_loading_error:
- typer.secho(f"Failed to validate: {library_loading_error}", fg=typer.colors.RED, err=True)
- present_validation_error(details_provider=library_loading_error)
- raise typer.Exit(1) from library_loading_error
+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION)
- with new_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
- if bundle_path:
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} bundle")
- else:
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} pipe")
-
- asyncio.run(validate_pipe(pipe_code=pipe_code, bundle_path=bundle_path))
-
-
-def present_validation_error(details_provider: ValidationErrorDetailsProtocol):
- console.print(details_provider)
- concept_definition_errors = details_provider.get_concept_definition_errors()
- if not concept_definition_errors:
- return
- for concept_definition_error in concept_definition_errors:
- syntax_error_data = concept_definition_error.structure_class_syntax_error_data
- if not syntax_error_data:
- continue
- message = concept_definition_error.message
- code = concept_definition_error.structure_class_python_code or ""
- highlight_lines: set[int] | None = None
- if syntax_error_data.lineno:
- highlight_lines = {syntax_error_data.lineno}
- syntax = Syntax(
- code=code,
- lexer="python",
- line_numbers=True,
- word_wrap=False,
- # theme="monokai", # pick any theme rich knows; omit to use default
- theme="ansi_dark", # pick any theme rich knows; omit to use default
- line_range=None,
- highlight_lines=highlight_lines,
- )
- pretty_range = ""
- if syntax_error_data.lineno and syntax_error_data.end_lineno:
- pretty_range = f"lines {syntax_error_data.lineno} to {syntax_error_data.end_lineno}"
- elif syntax_error_data.lineno:
- pretty_range = f"line {syntax_error_data.lineno}"
- if syntax_error_data.offset and syntax_error_data.end_offset:
- pretty_range += f", column {syntax_error_data.offset} to {syntax_error_data.end_offset}"
- elif syntax_error_data.offset:
- pretty_range += f", column {syntax_error_data.offset}"
- console.print(message)
- if pretty_range:
- pretty_print(f"Generated code error at {pretty_range}")
- console.print(syntax)
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())
+ if bundle_path:
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} bundle")
+ else:
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} pipe")
+
+ asyncio.run(validate_pipe(pipe_code=pipe_code, bundle_path=bundle_path))
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.VALIDATION)
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.VALIDATION)
+ finally:
+ pipelex_instance.teardown()
diff --git a/pipelex/cli/error_handlers.py b/pipelex/cli/error_handlers.py
new file mode 100644
index 000000000..377754865
--- /dev/null
+++ b/pipelex/cli/error_handlers.py
@@ -0,0 +1,181 @@
+"""Helper functions for handling and displaying CLI errors."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import TYPE_CHECKING, NoReturn
+
+import typer
+from rich.console import Console
+from rich.syntax import Syntax
+
+from pipelex.urls import URLs
+
+if TYPE_CHECKING:
+ from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
+ from pipelex.core.validation_errors import PipelexValidationExceptionAbstract
+ from pipelex.exceptions import PipeOperatorModelAvailabilityError, PipeOperatorModelChoiceError
+
+
+class ErrorContext(StrEnum):
+ """Context for error messages in CLI commands."""
+
+ PIPE_RUN = "Pipe run"
+ VALIDATION = "Pipe validation"
+ BUILD = "Pipe build"
+
+ # Pre-validation contexts (for Pipelex.make() errors)
+ VALIDATION_BEFORE_SHOW_PIPES = "Pre-validation (show pipes)"
+ VALIDATION_BEFORE_SHOW_PIPE = "Pre-validation (show pipe)"
+ VALIDATION_BEFORE_SHOW_MODELS = "Pre-validation (show models)"
+ VALIDATION_BEFORE_SHOW_BACKENDS = "Pre-validation (show backends)"
+ VALIDATION_BEFORE_PIPE_RUN = "Pre-validation (pipe run)"
+ VALIDATION_BEFORE_BUILD_PIPE = "Pre-validation (build pipe)"
+ VALIDATION_BEFORE_BUILD_RUNNER = "Pre-validation (build runner)"
+ VALIDATION_BEFORE_BUILD_INPUTS = "Pre-validation (build inputs)"
+ VALIDATION_BEFORE_BUILD_ONE_SHOT = "Pre-validation (build one-shot)"
+ VALIDATION_BEFORE_BUILD_PARTIAL = "Pre-validation (build partial)"
+
+
+def handle_model_choice_error(exc: PipeOperatorModelChoiceError, context: ErrorContext) -> NoReturn:
+ """Handle and display PipeOperatorModelChoiceError with formatted output.
+
+ Args:
+ exc: The model choice error exception
+ context: Context for the error message
+ """
+ console = Console(stderr=True)
+ console.print(f"\n[bold red]❌ {context} failed because of a model choice could not be interpreted correctly[/bold red]\n")
+ console.print(f"[bold cyan]Pipe:[/bold cyan] [yellow]'{exc.pipe_code}'[/yellow] [dim]({exc.pipe_type})[/dim]")
+ console.print(f"[bold cyan]Model Type:[/bold cyan] [yellow]'{exc.model_type}'[/yellow]")
+ console.print(f"[bold cyan]Model Choice:[/bold cyan] [yellow]'{exc.model_choice}'[/yellow]")
+ console.print(f"\n[bold red]Error:[/bold red] {exc.message}\n")
+ console.print(
+ f"[bold green]💡 Tip:[/bold green] Check your model configuration in [cyan].pipelex/inference/[/cyan] "
+ f"or specify a different model in the [yellow]'{exc.pipe_code}'[/yellow] pipe."
+ )
+ console.print(f"[dim]Learn more about the inference backend system: {URLs.backend_provider_docs}[/dim]")
+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")
+ raise typer.Exit(1) from exc
+
+
+def handle_model_availability_error(exc: PipeOperatorModelAvailabilityError, context: ErrorContext) -> NoReturn:
+ """Handle and display PipeOperatorModelAvailabilityError with formatted output.
+
+ Args:
+ exc: The model availability error exception
+ context: Context for the error message
+ """
+ console = Console(stderr=True)
+ console.print(f"\n[bold red]❌ {context} failed because a model wasn't available[/bold red]\n")
+ console.print(f"[bold cyan]Pipe:[/bold cyan] [yellow]'{exc.pipe_code}'[/yellow] [dim]({exc.pipe_type})[/dim]")
+ console.print(f"[bold cyan]Model:[/bold cyan] [yellow]'{exc.model_handle}'[/yellow]")
+ if exc.fallback_list:
+ fallbacks_str = ", ".join([f"[yellow]{fb}[/yellow]" for fb in exc.fallback_list])
+ console.print(f"[bold cyan]Fallbacks:[/bold cyan] {fallbacks_str}")
+ if len(exc.pipe_stack) > 1:
+ stack_str = " [dim]→[/dim] ".join([f"[yellow]{p}[/yellow]" for p in exc.pipe_stack])
+ console.print(f"[bold cyan]Pipe Stack:[/bold cyan] {stack_str}")
+ console.print(f"\n[bold red]Error:[/bold red] {exc}\n")
+ console.print(
+ f"[bold green]💡 Tip:[/bold green] Check your model configuration in [cyan].pipelex/inference/[/cyan] "
+ f"or specify a different model in the [yellow]'{exc.pipe_code}'[/yellow] pipe."
+ )
+ console.print(f"[dim]Learn more about the inference backend system: {URLs.backend_provider_docs}[/dim]")
+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")
+ raise typer.Exit(1) from exc
+
+
+def handle_model_deck_preset_error(exc: ModelDeckPresetValidatonError, context: ErrorContext) -> NoReturn:
+ """Handle and display ModelDeckPresetValidatonError with formatted output.
+
+ Args:
+ exc: The model deck preset validation error exception
+ context: Context for the error message
+ """
+ console = Console(stderr=True)
+ console.print(f"\n[bold red]❌ {context} failed due to model deck preset validation error[/bold red]\n")
+ console.print(f"[bold cyan]Preset ID:[/bold cyan] [yellow]'{exc.preset_id}'[/yellow]")
+ console.print(f"[bold cyan]Model Type:[/bold cyan] [yellow]'{exc.model_type}'[/yellow]")
+ console.print(f"[bold cyan]Model Handle:[/bold cyan] [yellow]'{exc.model_handle}'[/yellow]")
+ if exc.enabled_backends:
+ backends_str = ", ".join([f"[yellow]{b}[/yellow]" for b in sorted(exc.enabled_backends)])
+ console.print(f"[bold cyan]Enabled Backends:[/bold cyan] {backends_str}")
+ console.print(f"\n[bold red]Error:[/bold red] {exc.message}\n")
+ backends_str = ", ".join([f"[yellow]{b}[/yellow]" for b in sorted(exc.enabled_backends)])
+ console.print(
+ f"[bold green]💡 Tip:[/bold green] The preset [yellow]'{exc.preset_id}'[/yellow] references model handle "
+ f"[yellow]'{exc.model_handle}'[/yellow] which is not available in any enabled backend.\n"
+ f"The enabled backends are: {backends_str}."
+ )
+ console.print(
+ "[bold]Possible solutions:[/bold]\n"
+ " 1. Update the preset to use a different model\n"
+ f" 2. Configure model '{exc.model_handle}' in one of your enabled backends\n"
+ f" 3. Enable a backend that supports [yellow]'{exc.model_handle}'[/yellow]"
+ )
+ console.print(f"\n[dim]Learn more about the inference backend system: {URLs.backend_provider_docs}[/dim]")
+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")
+ raise typer.Exit(1) from exc
+
+
+def handle_validation_error(exc: PipelexValidationExceptionAbstract, context: ErrorContext) -> NoReturn:
+ """Handle and display validation errors with formatted output.
+
+ Args:
+ exc: The validation error exception (implements ValidationErrorDetailsProtocol)
+ context: Context for the error message
+ """
+ console = Console(stderr=True)
+ console.print(f"\n[bold red]❌ {context} failed due to validation error[/bold red]\n")
+ console.print(f"[bold red]Error:[/bold red] {exc}\n")
+
+ # Display concept definition errors with syntax highlighting
+ concept_definition_errors = exc.get_concept_definition_errors()
+ if concept_definition_errors:
+ console.print("[bold cyan]Concept Definition Errors:[/bold cyan]\n")
+ for concept_definition_error in concept_definition_errors:
+ syntax_error_data = concept_definition_error.structure_class_syntax_error_data
+ if not syntax_error_data:
+ continue
+
+ message = concept_definition_error.message
+ code = concept_definition_error.structure_class_python_code or ""
+ highlight_lines: set[int] | None = None
+ if syntax_error_data.lineno:
+ highlight_lines = {syntax_error_data.lineno}
+
+ syntax = Syntax(
+ code=code,
+ lexer="python",
+ line_numbers=True,
+ word_wrap=False,
+ theme="ansi_dark",
+ line_range=None,
+ highlight_lines=highlight_lines,
+ )
+
+ # Build pretty error location
+ pretty_range = ""
+ if syntax_error_data.lineno and syntax_error_data.end_lineno:
+ pretty_range = f"lines {syntax_error_data.lineno} to {syntax_error_data.end_lineno}"
+ elif syntax_error_data.lineno:
+ pretty_range = f"line {syntax_error_data.lineno}"
+ if syntax_error_data.offset and syntax_error_data.end_offset:
+ pretty_range += f", column {syntax_error_data.offset} to {syntax_error_data.end_offset}"
+ elif syntax_error_data.offset:
+ pretty_range += f", column {syntax_error_data.offset}"
+
+ console.print(f"[yellow]{message}[/yellow]")
+ if pretty_range:
+ console.print(f"[dim]Generated code error at {pretty_range}[/dim]")
+ console.print(syntax)
+ console.print()
+
+ console.print(
+ "[bold green]💡 Tip:[/bold green] Review the error message above and check your pipeline configuration. "
+ "Make sure all required fields are present and correctly formatted."
+ )
+ console.print(f"[dim]Learn more: {URLs.documentation}[/dim]")
+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")
+ raise typer.Exit(1) from exc
diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.py
index c0a4229d4..a1d265434 100644
--- a/pipelex/cogt/config_cogt.py
+++ b/pipelex/cogt/config_cogt.py
@@ -1,9 +1,9 @@
from pipelex.cogt.exceptions import LLMConfigError
from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobParamsDefaults
from pipelex.cogt.llm.llm_job_components import LLMJobConfig
+from pipelex.cogt.models.model_deck_config import ModelDeckConfig
from pipelex.plugins.fal.fal_config import FalConfig
from pipelex.system.configuration.config_model import ConfigModel
-from pipelex.tools.misc.file_utils import find_files_in_dir
class ExtractConfig(ConfigModel):
@@ -44,43 +44,8 @@ class LLMConfig(ConfigModel):
return template
-class InferenceManagerConfig(ConfigModel):
- is_auto_setup_preset_llm: bool
- is_auto_setup_preset_img_gen: bool
- is_auto_setup_preset_extract: bool
-
-
-class InferenceConfig(ConfigModel):
- inference_config_path: str
-
- @property
- def routing_profile_library_path(self) -> str:
- return f"{self.inference_config_path}/routing_profiles.toml"
-
- @property
- def backends_library_path(self) -> str:
- return f"{self.inference_config_path}/backends.toml"
-
- def model_specs_path(self, backend_name: str) -> str:
- return f"{self.inference_config_path}/backends/{backend_name}.toml"
-
- def get_model_deck_paths(self) -> list[str]:
- """Get all Model deck TOML file paths sorted alphabetically."""
- model_deck_paths = [
- str(path)
- for path in find_files_in_dir(
- dir_path=f"{self.inference_config_path}/deck",
- pattern="*.toml",
- is_recursive=True,
- )
- ]
- model_deck_paths.sort()
- return model_deck_paths
-
-
class Cogt(ConfigModel):
- inference_config: InferenceConfig
- inference_manager_config: InferenceManagerConfig
+ model_deck_config: ModelDeckConfig
llm_config: LLMConfig
img_gen_config: ImgGenConfig
extract_config: ExtractConfig
diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.py
index 5deb982ff..ac170c6e4 100644
--- a/pipelex/cogt/exceptions.py
+++ b/pipelex/cogt/exceptions.py
@@ -1,3 +1,7 @@
+from pipelex.cogt.extract.extract_setting import ExtractModelChoice
+from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice
+from pipelex.cogt.llm.llm_setting import LLMModelChoice
+from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.system.exceptions import FatalError, RootException
from pipelex.types import StrEnum
@@ -38,16 +42,11 @@ class LLMWorkerError(CogtError):
pass
-class LLMChoiceNotFoundError(CogtError):
- pass
-
-
-class ExtractChoiceNotFoundError(CogtError):
- pass
-
-
-class ImgGenChoiceNotFoundError(CogtError):
- pass
+class ModelChoiceNotFoundError(CogtError):
+ def __init__(self, message: str, model_type: ModelType, model_choice: LLMModelChoice | ExtractModelChoice | ImgGenModelChoice):
+ self.model_type = model_type
+ self.model_choice = model_choice
+ super().__init__(message=message)
class LLMSettingsValidationError(CogtError):
@@ -62,12 +61,55 @@ class ModelDeckValidatonError(CogtError):
pass
+class ModelDeckPresetValidatonError(ModelDeckValidatonError):
+ def __init__(
+ self,
+ message: str,
+ model_type: ModelType,
+ preset_id: str,
+ model_handle: str,
+ enabled_backends: set[str] | None = None,
+ ):
+ self.model_type = model_type
+ self.preset_id = preset_id
+ self.model_handle = model_handle
+ self.enabled_backends = enabled_backends or set()
+ super().__init__(message)
+
+
class ModelNotFoundError(CogtError):
- pass
+ def __init__(self, message: str, model_handle: str):
+ self.model_handle = model_handle
+ super().__init__(message)
+
+
+class ModelWaterfallError(ModelNotFoundError):
+ def __init__(self, message: str, model_handle: str, fallback_list: list[str]):
+ self.model_handle = model_handle
+ self.fallback_list = fallback_list
+ super().__init__(message=message, model_handle=model_handle)
class LLMHandleNotFoundError(CogtError):
- pass
+ def __init__(self, message: str, preset_id: str, model_handle: str, enabled_backends: set[str] | None = None):
+ self.preset_id = preset_id
+ self.model_handle = model_handle
+ self.enabled_backends = enabled_backends or set()
+ super().__init__(message)
+
+
+class ImgGenHandleNotFoundError(CogtError):
+ def __init__(self, message: str, preset_id: str, model_handle: str):
+ self.preset_id = preset_id
+ self.model_handle = model_handle
+ super().__init__(message)
+
+
+class ExtractHandleNotFoundError(CogtError):
+ def __init__(self, message: str, preset_id: str, model_handle: str):
+ self.preset_id = preset_id
+ self.model_handle = model_handle
+ super().__init__(message)
class LLMModelPlatformError(ValueError, CogtError):
@@ -163,7 +205,7 @@ class RoutingProfileLibraryNotFoundError(CogtError):
pass
-class RoutingProfileValidationError(CogtError):
+class RoutingProfileBlueprintValueError(CogtError, ValueError):
pass
@@ -211,7 +253,7 @@ class InferenceBackendLibraryError(CogtError):
pass
-class RoutingProfileError(CogtError):
+class RoutingProfileDisabledBackendError(CogtError):
pass
diff --git a/pipelex/cogt/img_gen/img_gen_setting.py b/pipelex/cogt/img_gen/img_gen_setting.py
index e2708914a..124f31ad0 100644
--- a/pipelex/cogt/img_gen/img_gen_setting.py
+++ b/pipelex/cogt/img_gen/img_gen_setting.py
@@ -2,7 +2,6 @@ from typing import Union
from pydantic import Field, model_validator
-from pipelex.cogt.exceptions import ImgGenSettingsValidationError
from pipelex.cogt.img_gen.img_gen_job_components import Quality
from pipelex.system.configuration.config_model import ConfigModel
from pipelex.types import Self
@@ -20,7 +19,7 @@ class ImgGenSetting(ConfigModel):
def validate_quality_or_nb_steps(self) -> Self:
if self.quality is not None and self.nb_steps is not None:
msg = "ImgGenSetting cannot have both 'quality' and 'nb_steps' specified. Use one or the other."
- raise ImgGenSettingsValidationError(msg)
+ raise ValueError(msg)
return self
def desc(self) -> str:
diff --git a/pipelex/cogt/inference/inference_manager.py b/pipelex/cogt/inference/inference_manager.py
index c2cf6a9c0..38b405b74 100644
--- a/pipelex/cogt/inference/inference_manager.py
+++ b/pipelex/cogt/inference/inference_manager.py
@@ -1,7 +1,6 @@
from typing_extensions import override
from pipelex import log
-from pipelex.cogt.exceptions import InferenceManagerWorkerSetupError
from pipelex.cogt.extract.extract_worker_abstract import ExtractWorkerAbstract
from pipelex.cogt.extract.extract_worker_factory import ExtractWorkerFactory
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
@@ -10,8 +9,6 @@ from pipelex.cogt.inference.inference_manager_protocol import InferenceManagerPr
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.llm_worker_factory import LLMWorkerFactory
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
-from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
-from pipelex.config import get_config
from pipelex.hub import get_models_manager, get_report_delegate
@@ -59,9 +56,9 @@ class InferenceManager(InferenceManagerProtocol):
def _setup_one_internal_llm_worker(
self,
- inference_model: InferenceModelSpec,
llm_handle: str,
) -> LLMWorkerInternalAbstract:
+ inference_model = get_models_manager().get_inference_model(model_handle=llm_handle)
llm_worker = LLMWorkerFactory.make_llm_worker(
inference_model=inference_model,
reporting_delegate=get_report_delegate(),
@@ -71,17 +68,10 @@ class InferenceManager(InferenceManagerProtocol):
@override
def get_llm_worker(self, llm_handle: str) -> LLMWorkerAbstract:
- if llm_worker := self.llm_workers.get(llm_handle):
- return llm_worker
- if not get_config().cogt.inference_manager_config.is_auto_setup_preset_llm:
- msg = f"No LLM worker for '{llm_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_llm"
- raise InferenceManagerWorkerSetupError(msg)
-
- inference_model = get_models_manager().get_inference_model(model_handle=llm_handle)
- return self._setup_one_internal_llm_worker(
- inference_model=inference_model,
- llm_handle=llm_handle,
- )
+ llm_worker = self.llm_workers.get(llm_handle)
+ if llm_worker is None:
+ llm_worker = self._setup_one_internal_llm_worker(llm_handle=llm_handle)
+ return llm_worker
@override
def set_llm_worker_from_external_plugin(
@@ -112,10 +102,6 @@ class InferenceManager(InferenceManagerProtocol):
def get_img_gen_worker(self, img_gen_handle: str) -> ImgGenWorkerAbstract:
img_gen_worker = self.img_gen_workers.get(img_gen_handle)
if img_gen_worker is None:
- if not get_config().cogt.inference_manager_config.is_auto_setup_preset_img_gen:
- msg = f"Found no ImgGen worker for '{img_gen_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_img_gen"
- raise InferenceManagerWorkerSetupError(msg)
-
img_gen_worker = self._setup_one_img_gen_worker(img_gen_handle=img_gen_handle)
return img_gen_worker
@@ -125,9 +111,9 @@ class InferenceManager(InferenceManagerProtocol):
def _setup_one_extract_worker(
self,
- inference_model: InferenceModelSpec,
extract_handle: str,
) -> ExtractWorkerAbstract:
+ inference_model = get_models_manager().get_inference_model(model_handle=extract_handle)
extract_worker = self.extract_worker_factory.make_extract_worker(
inference_model=inference_model,
reporting_delegate=get_report_delegate(),
@@ -137,14 +123,7 @@ class InferenceManager(InferenceManagerProtocol):
@override
def get_extract_worker(self, extract_handle: str) -> ExtractWorkerAbstract:
- if extract_worker := self.extract_workers.get(extract_handle):
- return extract_worker
- if not get_config().cogt.inference_manager_config.is_auto_setup_preset_extract:
- msg = f"Found no Extract worker for '{extract_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_extract"
- raise InferenceManagerWorkerSetupError(msg)
-
- inference_model = get_models_manager().get_inference_model(model_handle=extract_handle)
- return self._setup_one_extract_worker(
- inference_model=inference_model,
- extract_handle=extract_handle,
- )
+ extract_worker = self.extract_workers.get(extract_handle)
+ if extract_worker is None:
+ extract_worker = self._setup_one_extract_worker(extract_handle=extract_handle)
+ return extract_worker
diff --git a/pipelex/cogt/llm/llm_setting.py b/pipelex/cogt/llm/llm_setting.py
index d40d0d364..5e5933b41 100644
--- a/pipelex/cogt/llm/llm_setting.py
+++ b/pipelex/cogt/llm/llm_setting.py
@@ -48,7 +48,7 @@ class LLMSettingChoices(ConfigModel):
for_text: LLMModelChoice | None
for_object: LLMModelChoice | None
- def list_choices(self) -> set[str]:
+ def list_choice_strings(self) -> set[str]:
return {c for c in (self.for_text, self.for_object) if isinstance(c, str)}
@classmethod
diff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.py
index 9e956e421..0317e9702 100644
--- a/pipelex/cogt/model_backends/backend_library.py
+++ b/pipelex/cogt/model_backends/backend_library.py
@@ -1,3 +1,4 @@
+from functools import partial
from typing import TYPE_CHECKING, Any, cast
from pydantic import Field, RootModel, ValidationError
@@ -11,16 +12,31 @@ from pipelex.cogt.exceptions import (
InferenceModelSpecError,
)
from pipelex.cogt.model_backends.backend import InferenceBackend
-from pipelex.cogt.model_backends.backend_factory import InferenceBackendBlueprint, InferenceBackendFactory
-from pipelex.cogt.model_backends.model_spec_factory import InferenceModelSpecBlueprint, InferenceModelSpecFactory
-from pipelex.config import get_config
+from pipelex.cogt.model_backends.backend_factory import (
+ InferenceBackendBlueprint,
+ InferenceBackendFactory,
+)
+from pipelex.cogt.model_backends.model_spec_factory import (
+ InferenceModelSpecBlueprint,
+ InferenceModelSpecFactory,
+)
from pipelex.system.configuration.config_model import ConfigModel
from pipelex.system.environment import get_optional_env
from pipelex.system.runtime import runtime_manager
-from pipelex.tools.misc.dict_utils import apply_to_strings_recursive, extract_vars_from_strings_recursive
+from pipelex.tools.misc.dict_utils import (
+ apply_to_strings_recursive,
+ extract_vars_from_strings_recursive,
+)
from pipelex.tools.misc.placeholder import value_is_placeholder
from pipelex.tools.misc.toml_utils import load_toml_from_path
-from pipelex.tools.secrets.secrets_utils import UnknownVarPrefixError, VarFallbackPatternError, VarNotFoundError, substitute_vars
+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
+from pipelex.tools.secrets.secrets_utils import (
+ UnknownVarPrefixError,
+ VarFallbackPatternError,
+ VarNotFoundError,
+ substitute_vars,
+)
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
from pipelex.types import Self
if TYPE_CHECKING:
@@ -64,16 +80,20 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
def make_empty(cls) -> Self:
return cls(root={})
- def load(self, include_disabled: bool = False):
- backends_library_path = get_config().cogt.inference_config.backends_library_path
+ def load(self, secrets_provider: SecretsProviderAbstract, backends_library_path: str, backends_dir_path: str, include_disabled: bool = False):
try:
backends_dict = load_toml_from_path(path=backends_library_path)
except FileNotFoundError as file_not_found_exc:
msg = f"Could not find inference backend library at '{backends_library_path}': {file_not_found_exc}"
raise InferenceBackendLibraryNotFoundError(msg) from file_not_found_exc
except ValidationError as exc:
- msg = f"Invalid inference backend library configuration in '{backends_library_path}': {exc}"
+ valiation_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Invalid inference backend library configuration in '{backends_library_path}': {valiation_error_msg}"
raise InferenceBackendLibraryValidationError(msg) from exc
+
+ # Create a partial function with the secrets provider bound
+ substitute_vars_with_provider = partial(substitute_vars, secrets_provider=secrets_provider)
+
for backend_name, backend_dict in backends_dict.items():
# We'll split the read settings into standard fields and extra config
standard_fields = InferenceBackendBlueprint.model_fields.keys()
@@ -85,7 +105,7 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
if runtime_manager.is_ci_testing and backend_name == "vertexai":
continue
try:
- inference_backend_blueprint_dict = apply_to_strings_recursive(inference_backend_blueprint_dict_raw, substitute_vars)
+ inference_backend_blueprint_dict = apply_to_strings_recursive(inference_backend_blueprint_dict_raw, substitute_vars_with_provider)
except VarFallbackPatternError as var_fallback_pattern_exc:
msg = f"Variable substitution failed due to a pattern error in file '{backends_library_path}':\n{var_fallback_pattern_exc}"
key_name = "unknown"
@@ -123,13 +143,11 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
extra_config[key] = inference_backend_blueprint_dict.pop(key)
backend_blueprint = InferenceBackendBlueprint.model_validate(inference_backend_blueprint_dict)
- path_to_model_specs_toml = get_config().cogt.inference_config.model_specs_path(backend_name=backend_name)
+ path_to_model_specs_toml = f"{backends_dir_path}/{backend_name}.toml"
try:
- model_specs_dict_raw = load_toml_from_path(
- path=path_to_model_specs_toml,
- )
+ model_specs_dict_raw = load_toml_from_path(path=path_to_model_specs_toml)
try:
- model_specs_dict = apply_to_strings_recursive(model_specs_dict_raw, substitute_vars)
+ model_specs_dict = apply_to_strings_recursive(model_specs_dict_raw, substitute_vars_with_provider)
except (VarNotFoundError, UnknownVarPrefixError) as exc:
msg = f"Variable substitution failed in file '{path_to_model_specs_toml}': {exc}"
raise InferenceModelSpecError(msg) from exc
@@ -262,3 +280,6 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
def get_inference_backend(self, backend_name: str) -> InferenceBackend | None:
return self.root.get(backend_name)
+
+ def all_enabled_backends(self) -> list[str]:
+ return [backend_name for backend_name, backend in self.root.items() if backend.enabled]
diff --git a/pipelex/cogt/model_routing/routing_models.py b/pipelex/cogt/model_routing/routing_models.py
index b289b8509..f0fe818d8 100644
--- a/pipelex/cogt/model_routing/routing_models.py
+++ b/pipelex/cogt/model_routing/routing_models.py
@@ -15,3 +15,4 @@ class BackendMatchForModel(BaseModel):
routing_profile_name: str
matching_method: BackendMatchingMethod
matched_pattern: str | None
+ fallback_order: list[str] | None = None
diff --git a/pipelex/cogt/model_routing/routing_profile.py b/pipelex/cogt/model_routing/routing_profile.py
index 7a14f01e4..ae7d6aeef 100644
--- a/pipelex/cogt/model_routing/routing_profile.py
+++ b/pipelex/cogt/model_routing/routing_profile.py
@@ -12,29 +12,40 @@ class RoutingProfile(ConfigModel):
description: str | None = None
default: str | None = None
routes: dict[str, str] = Field(default_factory=dict) # Pattern -> Backend mapping
+ optional_routes: dict[str, str] = Field(default_factory=dict)
+ fallback_order: list[str] | None = None # Ordered list of backends for fallback
- def get_backend_match_for_model(self, model_name: str) -> BackendMatchForModel | None:
+ def get_backend_match_for_model(self, enabled_backends: list[str], model_name: str) -> BackendMatchForModel | None:
"""Get the backend name for a given model name.
Args:
+ enabled_backends: List of enabled backends
model_name: Name of the model to route
Returns:
Backend name to use for this model
"""
+ possible_routes = self.routes
+ for pattern, backend in self.optional_routes.items():
+ if backend not in enabled_backends:
+ continue
+ possible_routes[pattern] = backend
+
# Check exact matches first
- if model_name in self.routes:
+ if (backend_name := possible_routes.get(model_name)) and (backend_name in enabled_backends):
return BackendMatchForModel(
model_name=model_name,
- backend_name=self.routes[model_name],
+ backend_name=possible_routes[model_name],
routing_profile_name=self.name,
matching_method=BackendMatchingMethod.EXACT_MATCH,
matched_pattern=None,
)
# Check pattern matches
- for pattern, backend in self.routes.items():
+ for pattern, backend in possible_routes.items():
+ if backend not in enabled_backends:
+ continue
if matches_wildcard_pattern(model_name, pattern):
return BackendMatchForModel(
model_name=model_name,
@@ -44,13 +55,26 @@ class RoutingProfile(ConfigModel):
matched_pattern=pattern,
)
- # Return default backend
- if self.default:
+ # Validate fallback_order if set
+ validated_fallback_order: list[str] | None = None
+ if self.fallback_order:
+ validated_fallback_order = [backend for backend in self.fallback_order if backend in enabled_backends]
+
+ # Determine primary backend for DEFAULT matching
+ primary_backend: str | None = None
+ if self.default and self.default in enabled_backends:
+ primary_backend = self.default
+ elif validated_fallback_order:
+ primary_backend = validated_fallback_order[0]
+
+ # Return default backend match if we have a primary backend
+ if primary_backend:
return BackendMatchForModel(
model_name=model_name,
- backend_name=self.default,
+ backend_name=primary_backend,
routing_profile_name=self.name,
matching_method=BackendMatchingMethod.DEFAULT,
matched_pattern=None,
+ fallback_order=validated_fallback_order,
)
return None
diff --git a/pipelex/cogt/model_routing/routing_profile_factory.py b/pipelex/cogt/model_routing/routing_profile_factory.py
index a3c6e71e7..e9e3816f6 100644
--- a/pipelex/cogt/model_routing/routing_profile_factory.py
+++ b/pipelex/cogt/model_routing/routing_profile_factory.py
@@ -1,7 +1,9 @@
-from pydantic import Field
+from pydantic import Field, model_validator
+from pipelex.cogt.exceptions import RoutingProfileBlueprintValueError
from pipelex.cogt.model_routing.routing_profile import RoutingProfile
from pipelex.system.configuration.config_model import ConfigModel
+from pipelex.types import Self
class RoutingProfileBlueprint(ConfigModel):
@@ -10,6 +12,16 @@ class RoutingProfileBlueprint(ConfigModel):
description: str
default: str | None = None
routes: dict[str, str] = Field(default_factory=dict)
+ optional_routes: dict[str, str] = Field(default_factory=dict)
+ fallback_order: list[str] | None = None
+
+ @model_validator(mode="after")
+ def validate_routes(self) -> Self:
+ for pattern in self.optional_routes:
+ if pattern in self.routes:
+ msg = f"Pattern '{pattern}' is both in routes and optional_routes"
+ raise RoutingProfileBlueprintValueError(msg)
+ return self
class RoutingProfileLibraryBlueprint(ConfigModel):
@@ -43,4 +55,5 @@ class RoutingProfileFactory:
description=blueprint.description,
default=blueprint.default,
routes=blueprint.routes,
+ fallback_order=blueprint.fallback_order,
)
diff --git a/pipelex/cogt/model_routing/routing_profile_library.py b/pipelex/cogt/model_routing/routing_profile_library.py
deleted file mode 100644
index 2186796a5..000000000
--- a/pipelex/cogt/model_routing/routing_profile_library.py
+++ /dev/null
@@ -1,99 +0,0 @@
-from pydantic import Field, RootModel, ValidationError
-
-from pipelex import log
-from pipelex.cogt.exceptions import RoutingProfileLibraryError, RoutingProfileLibraryNotFoundError, RoutingProfileValidationError
-from pipelex.cogt.model_routing.routing_models import BackendMatchForModel
-from pipelex.cogt.model_routing.routing_profile import RoutingProfile
-from pipelex.cogt.model_routing.routing_profile_factory import (
- RoutingProfileFactory,
- RoutingProfileLibraryBlueprint,
-)
-from pipelex.config import get_config
-from pipelex.tools.misc.toml_utils import load_toml_from_path
-from pipelex.types import Self
-
-RoutingProfileLibraryRoot = dict[str, RoutingProfile]
-
-
-class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):
- """Library for managing routing profile configurations."""
-
- root: RoutingProfileLibraryRoot = Field(default_factory=dict)
- _active_config: str | None = None
-
- @property
- def active_profile(self) -> RoutingProfile:
- if not self._active_config:
- msg = "No active routing profile loaded"
- raise RoutingProfileLibraryError(msg)
- if self._active_config not in self.root:
- msg = f"Active routing profile '{self._active_config}' not found in loaded routing profile library"
- raise RoutingProfileLibraryError(msg)
- return self.root[self._active_config]
-
- @classmethod
- def make_empty(cls) -> Self:
- return cls(root={})
-
- def reset(self) -> None:
- self.root = {}
-
- def load(self) -> None:
- """Load the routing profile library configuration from TOML file."""
- routing_profile_library_path = get_config().cogt.inference_config.routing_profile_library_path
-
- try:
- catalog_dict = load_toml_from_path(path=routing_profile_library_path)
- except FileNotFoundError as not_found_exc:
- msg = f"Could not find routing profile library at '{routing_profile_library_path}': {not_found_exc}"
- raise RoutingProfileLibraryNotFoundError(msg) from not_found_exc
-
- try:
- catalog_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)
- except ValidationError as exc:
- msg = f"Invalid routing profile library configuration in '{routing_profile_library_path}': {exc}"
- raise RoutingProfileValidationError(msg) from exc
-
- # Validate that the active config exists
- if catalog_blueprint.active not in catalog_blueprint.profiles:
- msg = f"Active profile '{catalog_blueprint.active}' not found in library. Available profiles: {list(catalog_blueprint.profiles.keys())}"
- raise RoutingProfileLibraryError(msg)
-
- # Load all profiles
- self.root = {}
- for config_name, config_blueprint in catalog_blueprint.profiles.items():
- self.root[config_name] = RoutingProfileFactory.make_routing_profile(
- name=config_name,
- blueprint=config_blueprint,
- )
- self._active_config = catalog_blueprint.active
-
- log.verbose(f"Loaded routing profile library with active profile: '{self._active_config}'")
- log.verbose(f"Available profiles: {list(self.root.keys())}")
-
- def get_backend_match_for_model_from_active_routing_profile(self, model_name: str) -> BackendMatchForModel | None:
- """Get the backend name for a given model.
-
- Args:
- model_name: Name of the model to route
-
- Returns:
- Backend name to use for this model
-
- Raises:
- RoutingProfileLibraryError: If no active profile is set or config not found
-
- """
- profile = self.active_profile
- return profile.get_backend_match_for_model(model_name)
-
- def list_routing_profile_names(self) -> list[str]:
- """Get a list of all available routing profile names."""
- return list(self.root.keys())
-
- def get_required_routing_profile(self, routing_profile_name: str) -> RoutingProfile:
- routing_profile = self.root.get(routing_profile_name)
- if not routing_profile:
- msg = f"Routing profile '{routing_profile_name}' not found in loaded routing profile library"
- raise RoutingProfileLibraryError(msg)
- return routing_profile
diff --git a/pipelex/cogt/model_routing/routing_profile_loader.py b/pipelex/cogt/model_routing/routing_profile_loader.py
new file mode 100644
index 000000000..435d45822
--- /dev/null
+++ b/pipelex/cogt/model_routing/routing_profile_loader.py
@@ -0,0 +1,61 @@
+from pydantic import ValidationError
+
+from pipelex import log
+from pipelex.cogt.exceptions import ModelManagerError, RoutingProfileLibraryError
+from pipelex.cogt.model_routing.routing_profile import RoutingProfile
+from pipelex.cogt.model_routing.routing_profile_factory import RoutingProfileFactory, RoutingProfileLibraryBlueprint
+from pipelex.tools.misc.toml_utils import load_toml_from_path
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+
+
+def load_active_routing_profile(routing_profile_library_path: str, enabled_backends: list[str]) -> RoutingProfile:
+ """Load the active routing profile from the routing profile library from TOML file."""
+ # Load the routing profile library from TOML file
+ try:
+ catalog_dict = load_toml_from_path(path=routing_profile_library_path)
+ except FileNotFoundError as not_found_exc:
+ msg = f"Could not find routing profile library at '{routing_profile_library_path}': {not_found_exc}"
+ raise ModelManagerError(msg) from not_found_exc
+
+ # Validate the routing profile library configuration
+ try:
+ routing_profile_library_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)
+ except ValidationError as exc:
+ valiation_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Invalid routing profile library configuration in '{routing_profile_library_path}': {valiation_error_msg}"
+ raise ModelManagerError(msg) from exc
+
+ # Validate that the active profile exists
+ profile_names = ", ".join(list(routing_profile_library_blueprint.profiles.keys()))
+ active_profile_name = routing_profile_library_blueprint.active
+ if active_profile_name not in profile_names:
+ msg = f"Active profile '{active_profile_name}' not found in profile routing library. Available profiles: {profile_names}"
+ raise ModelManagerError(msg)
+
+ # Load all profiles
+ active_profile_blueprint = routing_profile_library_blueprint.profiles[active_profile_name]
+ active_profile = RoutingProfileFactory.make_routing_profile(
+ name=active_profile_name,
+ blueprint=active_profile_blueprint,
+ )
+ if active_profile.default and active_profile.default not in enabled_backends:
+ msg = (
+ f"Default backend '{active_profile.default}' set for routing profile '{active_profile_name}' is not enabled. "
+ f"You must either enable backend '{active_profile.default}' or set a different default backend for profile '{active_profile_name}', "
+ "or select a different routing profile."
+ )
+ raise RoutingProfileLibraryError(msg)
+
+ # Raise error for routes that use disabled backends
+ for backend_name in active_profile.routes.values():
+ if backend_name not in enabled_backends:
+ msg = f"Backend '{backend_name}', required for profile '{active_profile_name}' is not enabled"
+ raise RoutingProfileLibraryError(msg)
+
+ seen_disabled_backends: set[str] = set()
+ for backend_name in active_profile.routes.values():
+ if backend_name not in enabled_backends and backend_name not in seen_disabled_backends:
+ msg = f"Backend '{backend_name}', required for profile '{active_profile_name}' is not enabled"
+ log.info(msg)
+ seen_disabled_backends.add(backend_name)
+ return active_profile
diff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.py
index 2c9c2d650..358820cac 100644
--- a/pipelex/cogt/models/model_deck.py
+++ b/pipelex/cogt/models/model_deck.py
@@ -1,16 +1,17 @@
-from typing import Union
-
-from pydantic import Field, field_validator, model_validator
+from pydantic import Field, PrivateAttr, field_validator, model_validator
from pipelex import log
+from pipelex.cogt.config_cogt import ModelDeckConfig
from pipelex.cogt.exceptions import (
- ExtractChoiceNotFoundError,
- ImgGenChoiceNotFoundError,
- LLMChoiceNotFoundError,
+ ExtractHandleNotFoundError,
+ ImgGenHandleNotFoundError,
LLMHandleNotFoundError,
LLMSettingsValidationError,
+ ModelChoiceNotFoundError,
+ ModelDeckPresetValidatonError,
ModelDeckValidatonError,
ModelNotFoundError,
+ ModelWaterfallError,
)
from pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSetting
from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSetting
@@ -22,14 +23,16 @@ from pipelex.cogt.llm.llm_setting import (
)
from pipelex.cogt.model_backends.model_constraints import ModelConstraints
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.system.configuration.config_model import ConfigModel
from pipelex.system.exceptions import ConfigValidationError
+from pipelex.system.runtime import ProblemReaction
+from pipelex.tools.misc.toml_utils import load_toml_from_path_if_exists
from pipelex.types import Self
+from pipelex.urls import URLs
LLM_PRESET_DISABLED = "disabled"
-Waterfall = Union[str, list[str]]
-
class LLMDeckBlueprint(ConfigModel):
presets: dict[str, LLMSetting] = Field(default_factory=dict)
@@ -51,7 +54,8 @@ class ImgGenDeckBlueprint(ConfigModel):
class ModelDeckBlueprint(ConfigModel):
- aliases: dict[str, Waterfall] = Field(default_factory=dict)
+ aliases: dict[str, str] = Field(default_factory=dict)
+ waterfalls: dict[str, list[str]] = Field(default_factory=dict)
llm: LLMDeckBlueprint
extract: ExtractDeckBlueprint
@@ -59,8 +63,13 @@ class ModelDeckBlueprint(ConfigModel):
class ModelDeck(ConfigModel):
+ model_deck_config: ModelDeckConfig
inference_models: dict[str, InferenceModelSpec] = Field(default_factory=dict)
- aliases: dict[str, Waterfall] = Field(default_factory=dict)
+ aliases: dict[str, str] = Field(default_factory=dict)
+ waterfalls: dict[str, list[str]] = Field(default_factory=dict)
+
+ # Track which model_handle fallback warnings have been logged to avoid duplicates
+ _logged_fallback_warnings: set[str] = PrivateAttr(default_factory=set[str])
llm_presets: dict[str, LLMSetting] = Field(default_factory=dict)
llm_choice_defaults: LLMSettingChoicesDefaults
@@ -75,20 +84,28 @@ class ModelDeck(ConfigModel):
img_gen_presets: dict[str, ImgGenSetting] = Field(default_factory=dict)
img_gen_choice_default: ImgGenModelChoice
- def check_llm_setting(
+ def is_model_handle_defined(self, model_handle: str) -> bool:
+ all_handles: set[str] = set()
+ all_handles.update(self.inference_models.keys())
+ all_handles.update(self.aliases.keys())
+ if self.model_deck_config.is_model_fallback_enabled:
+ all_handles.update(self.waterfalls.keys())
+ return model_handle in all_handles
+
+ def check_llm_choice(
self,
- llm_setting_or_preset_id: LLMModelChoice,
+ llm_choice: LLMModelChoice,
is_disabled_allowed: bool = False,
):
- if isinstance(llm_setting_or_preset_id, LLMSetting):
+ if isinstance(llm_choice, LLMSetting):
return
- preset_id: str = llm_setting_or_preset_id
+ preset_id: str = llm_choice
if preset_id in self.llm_presets:
return
if preset_id == LLM_PRESET_DISABLED and is_disabled_allowed:
return
- msg = f"llm preset id '{preset_id}' not found in deck"
- raise LLMChoiceNotFoundError(msg)
+ msg = f"LLM preset '{preset_id}' was not found in the model deck"
+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.LLM, model_choice=llm_choice)
def get_llm_setting(self, llm_choice: LLMModelChoice) -> LLMSetting:
if isinstance(llm_choice, LLMSetting):
@@ -98,8 +115,8 @@ class ModelDeck(ConfigModel):
return llm_preset
if self.is_handle_defined(model_handle=llm_choice):
return LLMSetting(model=llm_choice, temperature=0.7, max_tokens=None)
- msg = f"LLM choice '{llm_choice}' not found in deck"
- raise LLMChoiceNotFoundError(msg)
+ msg = f"LLM choice '{llm_choice}' was not found in the model deck"
+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.LLM, model_choice=llm_choice)
def get_extract_setting(self, extract_choice: ExtractModelChoice) -> ExtractSetting:
if isinstance(extract_choice, ExtractSetting):
@@ -109,8 +126,8 @@ class ModelDeck(ConfigModel):
return extract_preset
if self.is_handle_defined(model_handle=extract_choice):
return ExtractSetting(model=extract_choice)
- msg = f"Extract choice '{extract_choice}' not found in deck"
- raise ExtractChoiceNotFoundError(msg)
+ msg = f"Extract choice '{extract_choice}' was not found in the model deck"
+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.TEXT_EXTRACTOR, model_choice=extract_choice)
def get_img_gen_setting(self, img_gen_choice: ImgGenModelChoice) -> ImgGenSetting:
if isinstance(img_gen_choice, ImgGenSetting):
@@ -120,8 +137,8 @@ class ModelDeck(ConfigModel):
return img_gen_preset
if self.is_handle_defined(model_handle=img_gen_choice):
return ImgGenSetting(model=img_gen_choice)
- msg = f"Image generation choice '{img_gen_choice}' not found in deck"
- raise ImgGenChoiceNotFoundError(msg)
+ msg = f"Image generation choice '{img_gen_choice}' was not found in the model deck"
+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.IMG_GEN, model_choice=img_gen_choice)
@classmethod
def final_validate(cls, deck: Self):
@@ -166,61 +183,209 @@ class ModelDeck(ConfigModel):
@field_validator("llm_choice_overrides", mode="after")
@classmethod
- def validate_llm_choice_overrides(cls, value: LLMSettingChoices) -> LLMSettingChoices:
+ def validate_llm_choice_disabled_overrides(cls, value: LLMSettingChoices) -> LLMSettingChoices:
if value.for_text == LLM_PRESET_DISABLED:
value.for_text = None
if value.for_object == LLM_PRESET_DISABLED:
value.for_object = None
return value
+ @model_validator(mode="after")
+ def validate_llm_choice_overrides(self) -> Self:
+ for llm_preset_id in self.llm_choice_overrides.list_choice_strings():
+ self.check_llm_choice(llm_choice=llm_preset_id)
+ return self
+
def validate_llm_presets(self) -> Self:
for llm_preset_id, llm_setting in self.llm_presets.items():
- if llm_setting.model not in self.inference_models:
- msg = f"llm_handle '{llm_setting.model}' for llm_preset '{llm_preset_id}' not found in deck"
- raise LLMHandleNotFoundError(msg)
+ if not self.is_model_handle_defined(model_handle=llm_setting.model):
+ enabled_backends = {model.backend_name for model in self.inference_models.values()}
+ msg = f"LLM handle '{llm_setting.model}' for llm preset '{llm_preset_id}' was not found in the model deck"
+ raise LLMHandleNotFoundError(
+ message=msg,
+ preset_id=llm_preset_id,
+ model_handle=llm_setting.model,
+ enabled_backends=enabled_backends,
+ )
return self
- @model_validator(mode="after")
- def validate_llm_setting_overrides(self) -> Self:
- self._validate_llm_choices(llm_choices=self.llm_choice_overrides)
+ def validate_img_gen_presets(self) -> Self:
+ for img_gen_preset_id, img_gen_setting in self.img_gen_presets.items():
+ if not self.is_model_handle_defined(model_handle=img_gen_setting.model):
+ msg = f"Image generation handle '{img_gen_setting.model}' for preset '{img_gen_preset_id}' was not found in the model deck"
+ raise ImgGenHandleNotFoundError(
+ message=msg,
+ preset_id=img_gen_preset_id,
+ model_handle=img_gen_setting.model,
+ )
+ return self
+
+ def validate_extract_presets(self) -> Self:
+ for extract_preset_id, extract_setting in self.extract_presets.items():
+ if not self.is_model_handle_defined(model_handle=extract_setting.model):
+ msg = f"Extract handle '{extract_setting.model}' for extract preset '{extract_preset_id}' was not found in the model deck"
+ raise ExtractHandleNotFoundError(
+ message=msg,
+ preset_id=extract_preset_id,
+ model_handle=extract_setting.model,
+ )
return self
- def _validate_llm_choices(self, llm_choices: LLMSettingChoices):
- for llm_setting in llm_choices.list_choices():
- self.check_llm_setting(llm_setting_or_preset_id=llm_setting)
+ def validate_registered_models(self):
+ self.validate_inference_models()
+ try:
+ self.validate_llm_presets()
+ except LLMHandleNotFoundError as exc:
+ match self.model_deck_config.missing_presets_reaction:
+ case ProblemReaction.RAISE:
+ msg = f"Failed to validate all LLM presets: {exc}"
+ raise ModelDeckPresetValidatonError(
+ message=msg,
+ model_type=ModelType.LLM,
+ preset_id=exc.preset_id,
+ model_handle=exc.model_handle,
+ enabled_backends=exc.enabled_backends,
+ ) from exc
+ case ProblemReaction.LOG:
+ log.warning(f"LLM handle not found: {exc}")
+ case ProblemReaction.NONE:
+ pass
+ try:
+ self.validate_img_gen_presets()
+ except ImgGenHandleNotFoundError as exc:
+ match self.model_deck_config.missing_presets_reaction:
+ case ProblemReaction.RAISE:
+ msg = f"Failed to validate all ImgGen presets: {exc}"
+ raise ModelDeckPresetValidatonError(
+ message=msg,
+ model_type=ModelType.IMG_GEN,
+ preset_id=exc.preset_id,
+ model_handle=exc.model_handle,
+ ) from exc
+ case ProblemReaction.LOG:
+ log.warning(f"ImgGen handle not found: {exc}")
+ case ProblemReaction.NONE:
+ pass
+ try:
+ self.validate_extract_presets()
+ except ExtractHandleNotFoundError as exc:
+ match self.model_deck_config.missing_presets_reaction:
+ case ProblemReaction.RAISE:
+ msg = f"Failed to validate all Extract presets: {exc}"
+ raise ModelDeckPresetValidatonError(
+ message=msg,
+ model_type=ModelType.TEXT_EXTRACTOR,
+ preset_id=exc.preset_id,
+ model_handle=exc.model_handle,
+ ) from exc
+ case ProblemReaction.LOG:
+ log.warning(f"Extract handle not found: {exc}")
+ case ProblemReaction.NONE:
+ pass
+
+ def validate_inference_models(self):
+ for model_handle in self.inference_models:
+ self.get_required_inference_model(model_handle=model_handle)
+
+ def _is_model_available_in_backend(self, model_handle: str, backend_name: str) -> bool | None:
+ """Check if a model is available from a specific backend.
+
+ This is a low-level check that reads the backend TOML file directly,
+ so it works even if the backend is disabled. Best-effort: returns False
+ if the file can't be read or parsed.
+
+ Args:
+ model_handle: The model handle/name to check for
+ backend_name: The backend name (e.g., 'pipelex_inference')
+
+ Returns:
+ True if the model is defined in the backend's TOML file, False otherwise
+ """
+ backend_file_path = f".pipelex/inference/backends/{backend_name}.toml"
+ try:
+ backend_toml = load_toml_from_path_if_exists(backend_file_path)
+ if backend_toml is None:
+ return None
+ # Check if model_handle exists as a top-level key (section) in the TOML
+ # Exclude special sections like 'defaults'
+ return model_handle in backend_toml and model_handle != "defaults"
+ except Exception:
+ # Best-effort: if anything goes wrong, just return None
+ return None
def get_optional_inference_model(self, model_handle: str) -> InferenceModelSpec | None:
if inference_model := self.inference_models.get(model_handle):
return inference_model
- if redirection := self.aliases.get(model_handle):
- log.verbose(f"Redirection for '{model_handle}': {redirection}")
- if isinstance(redirection, str):
- alias_list = [redirection]
- else:
- alias_list = redirection
- for alias in alias_list:
- if inference_model := self.get_optional_inference_model(model_handle=alias):
+ if alias := self.aliases.get(model_handle):
+ log.verbose(f"Alias for '{model_handle}': {alias}")
+ return self.get_optional_inference_model(model_handle=alias)
+ if fallback_list := self.waterfalls.get(model_handle):
+ ideal_model_handle = fallback_list[0]
+ log.verbose(f"Fallback list for '{model_handle}': {fallback_list}")
+ for fallback_index, fallback in enumerate(fallback_list):
+ if fallback_index > 0 and not self.model_deck_config.is_model_fallback_enabled:
+ # Waterfall disabled, so we raise an error
+ fallback_list_str = " → ".join(fallback_list)
+ msg = (
+ f"Model handle '{model_handle}' is a waterfall (i.e. a list of models to try in order), which resolves to "
+ f"•[ {fallback_list_str} ]•, but model fallbacks are disabled "
+ f"so only the first item in the list, '{ideal_model_handle}', is acceptable but it was not found in the deck. "
+ f"You must enable model fallback in your .pipelex/pipelex.toml file to permit the following fallbacks, "
+ f"or enable a backend that supports '{ideal_model_handle}'. "
+ )
+ raise ModelNotFoundError(message=msg, model_handle=model_handle)
+ if inference_model := self.get_optional_inference_model(model_handle=fallback):
+ if fallback_index > 0:
+ # Only log if we haven't logged for this model_handle before
+ if model_handle not in self._logged_fallback_warnings:
+ # Waterfall success: we explain what happened in the logs
+ msg = (
+ f"Inference model fallback: '{ideal_model_handle}' was not found in the model deck, "
+ f"so it was replaced by '{fallback}'. "
+ f"As a consequence, the results of the workflow may not have the expected quality, "
+ f"and the workflow might fail due to feature limitations such as context window size, etc. "
+ f"Consider getting access to '{ideal_model_handle}'."
+ )
+ enabled_backends = {model.backend_name for model in self.inference_models.values()}
+ if "pipelex_inference" not in enabled_backends and self._is_model_available_in_backend(
+ model_handle=ideal_model_handle, backend_name="pipelex_inference"
+ ):
+ msg += (
+ f" Note that many high quality models such as '{ideal_model_handle}' are available from the Pipelex Inference "
+ f"platform and you can get free credits to try them out, please see our docs for more details about setting up "
+ f"Pipelex Inference or other inference backends:\n{URLs.backend_provider_docs}"
+ )
+ else:
+ msg += f" Please see our docs for more details about setting up inference backends:\n{URLs.backend_provider_docs}"
+ log.info(msg)
+ # Mark this warning as logged for this model_handle
+ self._logged_fallback_warnings.add(model_handle)
return inference_model
- log.warning(f"Skipping model handle '{model_handle}' because it's not found in deck, it could be an external plugin.")
+ msg = (
+ f"Model handle '{model_handle}' is a waterfall (i.e. a list of models to try in order) "
+ "but none of the fallback models were found in the model deck"
+ )
+ raise ModelWaterfallError(message=msg, model_handle=model_handle, fallback_list=fallback_list)
+ log.verbose(f"Skipping model handle '{model_handle}' because it's was not found in the model deck, it could be an external plugin.")
return None
def is_handle_defined(self, model_handle: str) -> bool:
- return model_handle in self.inference_models or model_handle in self.aliases
+ return model_handle in self.inference_models or model_handle in self.aliases or model_handle in self.waterfalls
def get_required_inference_model(self, model_handle: str) -> InferenceModelSpec:
inference_model = self.get_optional_inference_model(model_handle=model_handle)
if inference_model is None:
msg = (
- f"Model handle '{model_handle}' not found in deck. "
- "Make sure its defined in the model decks '.pipelex/inference/deck/base_deck.toml' or '.pipelex/inference/deck/overrides.toml'"
+ f"Model handle '{model_handle}' was not found in the model deck. "
+ "Make sure it's defined in ond of the model decks '.pipelex/inference/deck/*.toml'. "
"If the model handle is indeed in the deck, make sure the required backend for this model to run is enabled in "
"'.pipelex/inference/backends.toml' and that you have the necessary credentials."
- "To find what backend is required for this model, look at the routing profile in .pipelex/inference/routing_profiles.toml"
+ "To find what backend is required for this model, look at the routing profile in .pipelex/inference/routing_profiles.toml. "
"Learn more about the inference backend system in the Pipelex documentation: "
"https://docs.pipelex.com/pages/configuration/config-technical/inference-backend-config/"
)
- raise ModelNotFoundError(msg)
+ raise ModelNotFoundError(message=msg, model_handle=model_handle)
if model_handle not in self.inference_models:
log.verbose(f"Model handle '{model_handle}' is an alias which resolves to '{inference_model.name}'")
return inference_model
diff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.py
index 0efca195a..8cf96c6e9 100644
--- a/pipelex/cogt/models/model_deck_check.py
+++ b/pipelex/cogt/models/model_deck_check.py
@@ -1,7 +1,8 @@
-from pipelex.cogt.exceptions import ExtractChoiceNotFoundError, ImgGenChoiceNotFoundError, LLMChoiceNotFoundError
+from pipelex.cogt.exceptions import ModelChoiceNotFoundError
from pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSetting
from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSetting
from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting
+from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.hub import get_model_deck
@@ -13,8 +14,8 @@ def check_llm_choice_with_deck(llm_choice: LLMModelChoice):
if llm_choice in model_deck.llm_presets or model_deck.is_handle_defined(model_handle=llm_choice):
return
- msg = f"LLM choice '{llm_choice}' not found in deck"
- raise LLMChoiceNotFoundError(msg)
+ msg = f"LLM choice '{llm_choice}' was not found in the model deck"
+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.LLM, model_choice=llm_choice)
def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):
@@ -23,8 +24,8 @@ def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):
model_deck = get_model_deck()
if extract_choice in model_deck.extract_presets or model_deck.is_handle_defined(model_handle=extract_choice):
return
- msg = f"OCR choice '{extract_choice}' not found in deck"
- raise ExtractChoiceNotFoundError(msg)
+ msg = f"OCR choice '{extract_choice}' was not found in the model deck"
+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.TEXT_EXTRACTOR, model_choice=extract_choice)
def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):
@@ -33,5 +34,5 @@ def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):
model_deck = get_model_deck()
if img_gen_choice in model_deck.img_gen_presets or model_deck.is_handle_defined(model_handle=img_gen_choice):
return
- msg = f"Image generation choice '{img_gen_choice}' not found in deck"
- raise ImgGenChoiceNotFoundError(msg)
+ msg = f"Image generation choice '{img_gen_choice}' was not found in the model deck"
+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.IMG_GEN, model_choice=img_gen_choice)
diff --git a/pipelex/cogt/models/model_deck_config.py b/pipelex/cogt/models/model_deck_config.py
new file mode 100644
index 000000000..04d4bcad3
--- /dev/null
+++ b/pipelex/cogt/models/model_deck_config.py
@@ -0,0 +1,9 @@
+from pydantic import Field
+
+from pipelex.system.configuration.config_model import ConfigModel
+from pipelex.system.runtime import ProblemReaction
+
+
+class ModelDeckConfig(ConfigModel):
+ is_model_fallback_enabled: bool
+ missing_presets_reaction: ProblemReaction = Field(strict=False)
diff --git a/pipelex/cogt/models/model_deck_loader.py b/pipelex/cogt/models/model_deck_loader.py
new file mode 100644
index 000000000..908404e3f
--- /dev/null
+++ b/pipelex/cogt/models/model_deck_loader.py
@@ -0,0 +1,31 @@
+from typing import Any
+
+from pydantic import ValidationError
+
+from pipelex.cogt.exceptions import ModelDeckNotFoundError, ModelDeckValidationError
+from pipelex.cogt.models.model_deck import ModelDeckBlueprint
+from pipelex.tools.misc.json_utils import deep_update
+from pipelex.tools.misc.toml_utils import load_toml_from_path
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+
+
+def load_model_deck_blueprint(model_deck_paths: list[str]) -> ModelDeckBlueprint:
+ full_deck_dict: dict[str, Any] = {}
+ if not model_deck_paths:
+ msg = "No Model deck paths found. Please run `pipelex init config` to create the set up the base deck."
+ raise ModelDeckNotFoundError(msg)
+
+ for deck_path in model_deck_paths:
+ try:
+ deck_dict = load_toml_from_path(path=deck_path)
+ except FileNotFoundError as not_found_exc:
+ msg = f"Could not find Model Deck file at '{deck_path}': {not_found_exc}"
+ raise ModelDeckNotFoundError(msg) from not_found_exc
+ deep_update(full_deck_dict, deck_dict)
+
+ try:
+ return ModelDeckBlueprint.model_validate(full_deck_dict)
+ except ValidationError as exc:
+ valiation_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Invalid Model Deck configuration in {model_deck_paths}: {valiation_error_msg}"
+ raise ModelDeckValidationError(msg) from exc
diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.py
index a8f4cff27..1ddb954ad 100644
--- a/pipelex/cogt/models/model_manager.py
+++ b/pipelex/cogt/models/model_manager.py
@@ -1,25 +1,23 @@
-from typing import Any
-
-from pydantic import ValidationError
from typing_extensions import override
-from pipelex import log
-from pipelex.cogt.exceptions import ModelDeckNotFoundError, ModelDeckValidationError, ModelManagerError
+from pipelex.cogt.exceptions import ModelManagerError
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.backend_library import InferenceBackendLibrary
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_routing.routing_models import BackendMatchingMethod
-from pipelex.cogt.model_routing.routing_profile_library import RoutingProfileLibrary
+from pipelex.cogt.model_routing.routing_profile import RoutingProfile
+from pipelex.cogt.model_routing.routing_profile_loader import load_active_routing_profile
from pipelex.cogt.models.model_deck import ModelDeck, ModelDeckBlueprint
+from pipelex.cogt.models.model_deck_loader import load_model_deck_blueprint
from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract
-from pipelex.config import get_config
-from pipelex.tools.misc.json_utils import deep_update
-from pipelex.tools.misc.toml_utils import load_toml_from_path
+from pipelex.config import ConfigPaths, get_config
+from pipelex.tools.misc.file_utils import find_files_in_dir
+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
class ModelManager(ModelManagerAbstract):
def __init__(self) -> None:
- self.routing_profile_library = RoutingProfileLibrary.make_empty()
+ self._routing_profile: RoutingProfile | None = None
self.inference_backend_library = InferenceBackendLibrary.make_empty()
self.model_deck: ModelDeck | None = None
@@ -30,50 +28,63 @@ class ModelManager(ModelManagerAbstract):
raise RuntimeError(msg)
return self.model_deck
+ @classmethod
+ def get_model_deck_paths(cls, deck_dir_path: str) -> list[str]:
+ """Get all Model deck TOML file paths sorted alphabetically."""
+ model_deck_paths = [
+ str(path)
+ for path in find_files_in_dir(
+ dir_path=deck_dir_path,
+ pattern="*.toml",
+ is_recursive=True,
+ )
+ ]
+ model_deck_paths.sort()
+ return model_deck_paths
+
@override
def teardown(self) -> None:
- self.routing_profile_library.reset()
+ self.model_deck = None
self.inference_backend_library.reset()
+ self._routing_profile = None
@override
- def setup(self) -> None:
- self.routing_profile_library.load()
- self.inference_backend_library.load()
- deck_blueprint = self.load_deck_blueprint()
- self.model_deck = self.build_deck(model_deck_blueprint=deck_blueprint)
-
- @classmethod
- def load_deck_blueprint(cls) -> ModelDeckBlueprint:
- deck_paths = get_config().cogt.inference_config.get_model_deck_paths()
- full_deck_dict: dict[str, Any] = {}
- if not deck_paths:
- msg = "No Model deck paths found. Please run `pipelex init config` to create the set up the base deck."
- raise ModelDeckNotFoundError(msg)
+ def setup(self, secrets_provider: SecretsProviderAbstract) -> None:
+ self.inference_backend_library.load(
+ secrets_provider=secrets_provider,
+ backends_library_path=ConfigPaths.BACKENDS_FILE_PATH,
+ backends_dir_path=ConfigPaths.BACKENDS_DIR_PATH,
+ )
+ enabled_backends = self.inference_backend_library.all_enabled_backends()
+ self._routing_profile = load_active_routing_profile(
+ routing_profile_library_path=ConfigPaths.ROUTING_PROFILES_FILE_PATH,
+ enabled_backends=enabled_backends,
+ )
+ model_deck_paths = ModelManager.get_model_deck_paths(deck_dir_path=ConfigPaths.MODEL_DECKS_DIR_PATH)
+ deck_blueprint = load_model_deck_blueprint(model_deck_paths=model_deck_paths)
+ self.model_deck = self.build_deck(enabled_backends=enabled_backends, model_deck_blueprint=deck_blueprint)
- for deck_path in deck_paths:
- try:
- deck_dict = load_toml_from_path(path=deck_path)
- except FileNotFoundError as not_found_exc:
- msg = f"Could not find Model Deck file at '{deck_path}': {not_found_exc}"
- raise ModelDeckNotFoundError(msg) from not_found_exc
- deep_update(full_deck_dict, deck_dict)
+ @override
+ def validate_model_deck(self):
+ self.get_model_deck().validate_registered_models()
- try:
- return ModelDeckBlueprint.model_validate(full_deck_dict)
- except ValidationError as exc:
- msg = f"Invalid Model Deck configuration in {deck_paths}: {exc}"
- raise ModelDeckValidationError(msg) from exc
+ @property
+ def routing_profile(self) -> RoutingProfile:
+ if self._routing_profile is None:
+ msg = "No active routing profile loaded"
+ raise RuntimeError(msg)
+ return self._routing_profile
- def build_deck(self, model_deck_blueprint: ModelDeckBlueprint) -> ModelDeck:
+ def build_deck(self, enabled_backends: list[str], model_deck_blueprint: ModelDeckBlueprint) -> ModelDeck:
all_models_and_possible_backends = self.inference_backend_library.get_all_models_and_possible_backends()
inference_models: dict[str, InferenceModelSpec] = {}
for model_name, available_backends in all_models_and_possible_backends.items():
- backend_match_for_model = self.routing_profile_library.get_backend_match_for_model_from_active_routing_profile(
+ backend_match_for_model = self.routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
model_name=model_name,
)
if backend_match_for_model is None:
- log.verbose(f"No backend match found for model '{model_name}'")
continue
matched_backend_name = backend_match_for_model.backend_name
backend = self.inference_backend_library.get_inference_backend(backend_name=matched_backend_name)
@@ -91,17 +102,22 @@ class ModelManager(ModelManagerAbstract):
)
raise ModelManagerError(msg)
case BackendMatchingMethod.PATTERN_MATCH:
- log.verbose(
- f"Model spec '{model_name}' not found in backend '{matched_backend_name}' but it's OK because "
- f"it was only matched by pattern in routing profile '{backend_match_for_model.routing_profile_name}'",
- )
# We can skip it because it was only a pattern match
continue
case BackendMatchingMethod.DEFAULT:
# We could not find the model spec, but it was a default match,
# so we can look for it in the other available backends
- # TODO: enable to set the order or priority of the available backends
- for available_backend in available_backends:
+ # Use fallback_order if specified, then try remaining enabled backends
+ if backend_match_for_model.fallback_order:
+ # Try fallback_order first, then any enabled backends not in fallback_order
+ backends_to_try = backend_match_for_model.fallback_order + [
+ b for b in enabled_backends if b not in backend_match_for_model.fallback_order
+ ]
+ else:
+ # No fallback_order specified, use all available_backends
+ backends_to_try = available_backends
+
+ for available_backend in backends_to_try:
if available_backend == matched_backend_name:
# we've already checked the matched_backend_name and it didn't have the model spec, that's why we're here
continue
@@ -113,16 +129,15 @@ class ModelManager(ModelManagerAbstract):
if model_spec is not None:
break
if model_spec is None:
- msg = (
- f"Model spec '{model_name}' not found in any of the available backends '{available_backends}' "
- f"which was set as default in routing profile '{backend_match_for_model.routing_profile_name}'"
- )
- raise ModelManagerError(msg)
+ # Model not available in any of the searched backends - skip it
+ # Not all models need to be available in the configured backends
+ continue
inference_models[model_name] = model_spec
return ModelDeck(
inference_models=inference_models,
aliases=model_deck_blueprint.aliases,
+ waterfalls=model_deck_blueprint.waterfalls,
llm_presets=model_deck_blueprint.llm.presets,
llm_choice_defaults=model_deck_blueprint.llm.choice_defaults,
llm_choice_overrides=model_deck_blueprint.llm.choice_overrides,
@@ -130,6 +145,7 @@ class ModelManager(ModelManagerAbstract):
extract_choice_default=model_deck_blueprint.extract.choice_default,
img_gen_presets=model_deck_blueprint.img_gen.presets,
img_gen_choice_default=model_deck_blueprint.img_gen.choice_default,
+ model_deck_config=get_config().cogt.model_deck_config,
)
@override
diff --git a/pipelex/cogt/models/model_manager_abstract.py b/pipelex/cogt/models/model_manager_abstract.py
index b0cc94593..e97a35c64 100644
--- a/pipelex/cogt/models/model_manager_abstract.py
+++ b/pipelex/cogt/models/model_manager_abstract.py
@@ -3,15 +3,20 @@ from abc import ABC, abstractmethod
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.models.model_deck import ModelDeck
+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
class ModelManagerAbstract(ABC):
+ @abstractmethod
+ def validate_model_deck(self):
+ pass
+
@abstractmethod
def teardown(self) -> None:
pass
@abstractmethod
- def setup(self) -> None:
+ def setup(self, secrets_provider: SecretsProviderAbstract) -> None:
pass
@abstractmethod
diff --git a/pipelex/config.py b/pipelex/config.py
index f82415790..4937bb4ca 100644
--- a/pipelex/config.py
+++ b/pipelex/config.py
@@ -17,6 +17,24 @@ from pipelex.tools.log.log_config import LogConfig
from pipelex.types import StrEnum
+class ConfigPaths:
+ DEFAULT_CONFIG_DIR_PATH = "./.pipelex"
+ INFERENCE_DIR_NAME = "inference"
+ INFERENCE_DIR_PATH = f"{DEFAULT_CONFIG_DIR_PATH}/{INFERENCE_DIR_NAME}"
+ BACKENDS_FILE_NAME = "backends.toml"
+ BACKENDS_FILE_PATH = f"{INFERENCE_DIR_PATH}/{BACKENDS_FILE_NAME}"
+ BACKENDS_DIR_NAME = "backends"
+ BACKENDS_DIR_PATH = f"{INFERENCE_DIR_PATH}/{BACKENDS_DIR_NAME}"
+ ROUTING_PROFILES_FILE_NAME = "routing_profiles.toml"
+ ROUTING_PROFILES_FILE_PATH = f"{INFERENCE_DIR_PATH}/{ROUTING_PROFILES_FILE_NAME}"
+ MODEL_DECKS_DIR_NAME = "deck"
+ MODEL_DECKS_DIR_PATH = f"{INFERENCE_DIR_PATH}/{MODEL_DECKS_DIR_NAME}"
+ BASE_DECK_FILE_NAME = "base_deck.toml"
+ BASE_DECK_FILE_PATH = f"{MODEL_DECKS_DIR_PATH}/{BASE_DECK_FILE_NAME}"
+ OVERRIDES_DECK_FILE_NAME = "overrides.toml"
+ OVERRIDES_DECK_FILE_PATH = f"{MODEL_DECKS_DIR_PATH}/{OVERRIDES_DECK_FILE_NAME}"
+
+
class StaticValidationReaction(StrEnum):
RAISE = "raise"
LOG = "log"
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index 551d87746..44bc6e0e5 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -1,14 +1,15 @@
from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, final
from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pipelex.cogt.exceptions import ModelChoiceNotFoundError
from pipelex.core.concepts.concept import Concept
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.input_requirements import InputRequirements
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
from pipelex.core.pipes.pipe_output import PipeOutput
-from pipelex.exceptions import PipeStackOverflowError
+from pipelex.exceptions import PipeOperatorModelChoiceError, PipeStackOverflowError
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
@@ -38,8 +39,22 @@ class PipeAbstract(ABC, BaseModel):
def validate_output(self):
"""Validate the output for the pipe."""
+ @final
def validate_with_libraries(self):
"""Validate the pipe with the libraries, after the static validation"""
+ try:
+ self._validate_with_libraries()
+ except ModelChoiceNotFoundError as exc:
+ raise PipeOperatorModelChoiceError(
+ message=exc.message,
+ pipe_type=self.pipe_type,
+ pipe_code=self.code,
+ model_type=exc.model_type,
+ model_choice=exc.model_choice,
+ ) from exc
+
+ def _validate_with_libraries(self):
+ """Validate the pipe with the libraries, after the static validation"""
@abstractmethod
def required_variables(self) -> set[str]:
diff --git a/pipelex/core/validation_errors.py b/pipelex/core/validation_errors.py
index 50a9ce948..d5e8566d7 100644
--- a/pipelex/core/validation_errors.py
+++ b/pipelex/core/validation_errors.py
@@ -1,7 +1,9 @@
-from typing_extensions import Protocol
+from abc import ABC, abstractmethod
from pipelex.builder.validation_error_data import ConceptDefinitionErrorData
-class ValidationErrorDetailsProtocol(Protocol):
- def get_concept_definition_errors(self) -> list[ConceptDefinitionErrorData]: ...
+class PipelexValidationExceptionAbstract(Exception, ABC):
+ @abstractmethod
+ def get_concept_definition_errors(self) -> list[ConceptDefinitionErrorData]:
+ pass
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index ef83c534d..ac98bb825 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -12,11 +12,15 @@ from pipelex.builder.validation_error_data import (
StaticValidationErrorType,
SyntaxErrorData,
)
-from pipelex.core.validation_errors import ValidationErrorDetailsProtocol
+from pipelex.core.validation_errors import PipelexValidationExceptionAbstract
from pipelex.system.exceptions import RootException
from pipelex.tools.misc.context_provider_abstract import ContextProviderException
if TYPE_CHECKING:
+ from pipelex.cogt.extract.extract_setting import ExtractModelChoice
+ from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice
+ from pipelex.cogt.llm.llm_setting import LLMModelChoice
+ from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.templating.template_category import TemplateCategory
from pipelex.pipe_run.pipe_run_mode import PipeRunMode
@@ -127,7 +131,7 @@ class LibraryError(PipelexException):
pass
-class LibraryLoadingError(LibraryError, ValidationErrorDetailsProtocol):
+class LibraryLoadingError(LibraryError, PipelexValidationExceptionAbstract):
"""Error raised when loading library components fails."""
def __init__(
@@ -377,6 +381,65 @@ class StructureClassError(PipelexException):
pass
+class PipeOperatorException(PipelexException):
+ pass
+
+
+class PipeOperatorModelAvailabilityError(PipelexException):
+ def __init__(
+ self,
+ message: str,
+ run_mode: PipeRunMode,
+ pipe_type: str,
+ pipe_code: str,
+ pipe_stack: list[str],
+ model_handle: str,
+ fallback_list: list[str] | None = None,
+ ):
+ self.run_mode = run_mode
+ self.pipe_type = pipe_type
+ self.pipe_code = pipe_code
+ self.pipe_stack = pipe_stack
+ self.model_handle = model_handle
+ self.fallback_list = fallback_list
+ super().__init__(message)
+
+
+class PipeOperatorModelChoiceError(PipelexException):
+ def __init__(
+ self,
+ message: str,
+ pipe_type: str,
+ pipe_code: str,
+ model_type: ModelType,
+ model_choice: LLMModelChoice | ExtractModelChoice | ImgGenModelChoice,
+ ):
+ self.pipe_type = pipe_type
+ self.pipe_code = pipe_code
+ self.model_type = model_type
+ self.model_choice = model_choice
+ super().__init__(message)
+
+ def desc(self) -> str:
+ msg = f"{self.message}"
+ msg += f" • pipe='{self.pipe_code}' ({self.pipe_type})"
+ msg += f" • model_type='{self.model_type}'"
+
+ # Extract the choice identifier from the model_choice union type
+ if isinstance(self.model_choice, str):
+ # It's a preset/alias string
+ msg += f" • choice='{self.model_choice}'"
+ else:
+ # It's a Setting object with a model field and optional desc()
+ msg += f" • choice={self.model_choice.desc()}"
+
+ return msg
+
+ @override
+ def __str__(self) -> str:
+ return self.desc()
+
+
class PipeControllerError(PipelexException):
pass
diff --git a/pipelex/hub.py b/pipelex/hub.py
index 1af5c59b9..8e1015210 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -90,14 +90,14 @@ class PipelexHub:
# tools
- def setup_config(self, config_cls: type[ConfigRoot], specific_config_path: str | None = None):
+ def setup_config(self, config_cls: type[ConfigRoot]):
"""Set the global configuration instance.
# Args:
# config (Config): The configuration instance to set.
"""
- config = config_manager.load_config(specific_config_path)
- self.set_config(config=config_cls.model_validate(config))
+ config_dict = config_manager.load_config()
+ self.set_config(config=config_cls.model_validate(config_dict))
def set_config(self, config: ConfigRoot):
if self._config is not None:
diff --git a/pipelex/kit/agent_rules/llms.md b/pipelex/kit/agent_rules/llms.md
index 1c3b01521..e8b5979ed 100644
--- a/pipelex/kit/agent_rules/llms.md
+++ b/pipelex/kit/agent_rules/llms.md
@@ -48,7 +48,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/pipelex/kit/agent_rules/write_pipelex.md b/pipelex/kit/agent_rules/write_pipelex.md
index 77405af0a..aa4110624 100644
--- a/pipelex/kit/agent_rules/write_pipelex.md
+++ b/pipelex/kit/agent_rules/write_pipelex.md
@@ -817,7 +817,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp
Examples:
```toml
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
+llm_to_engineer = { model = "base-claude", temperature = 1 }
llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }
```
diff --git a/pipelex/kit/configs/inference/backends.toml b/pipelex/kit/configs/inference/backends.toml
index 027d9fee0..536f3675d 100644
--- a/pipelex/kit/configs/inference/backends.toml
+++ b/pipelex/kit/configs/inference/backends.toml
@@ -16,6 +16,11 @@ enabled = true
endpoint = "https://inference.pipelex.com/v1"
api_key = "${PIPELEX_INFERENCE_API_KEY}"
+[anthropic]
+enabled = false
+api_key = "${ANTHROPIC_API_KEY}"
+claude_4_tokens_limit = 8192
+
[azure_openai]
display_name = "Azure OpenAI"
enabled = false
@@ -28,32 +33,42 @@ display_name = "Amazon Bedrock"
enabled = false
aws_region = "${AWS_REGION}"
+[blackboxai]
+display_name = "BlackBox AI"
+enabled = false
+endpoint = "https://api.blackbox.ai/v1"
+api_key = "${BLACKBOX_API_KEY}"
+
+[fal]
+display_name = "FAL"
+enabled = false
+api_key = "${FAL_API_KEY}"
+
[google]
display_name = "Google AI"
enabled = false
api_key = "${GOOGLE_API_KEY}"
-[vertexai]
-display_name = "Google Vertex AI"
+[mistral]
+display_name = "Mistral AI"
enabled = false
-gcp_project_id = "${GCP_PROJECT_ID}"
-gcp_location = "${GCP_LOCATION}"
-gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+api_key = "${MISTRAL_API_KEY}"
+
+[ollama]
+enabled = false
+endpoint = "http://localhost:11434/v1"
[openai]
display_name = "OpenAI"
enabled = false
api_key = "${OPENAI_API_KEY}"
-[anthropic]
-enabled = false
-api_key = "${ANTHROPIC_API_KEY}"
-claude_4_tokens_limit = 8192
-
-[mistral]
-display_name = "Mistral AI"
+[vertexai]
+display_name = "Google Vertex AI"
enabled = false
-api_key = "${MISTRAL_API_KEY}"
+gcp_project_id = "${GCP_PROJECT_ID}"
+gcp_location = "${GCP_LOCATION}"
+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
[xai]
display_name = "xAI"
@@ -61,20 +76,5 @@ enabled = false
endpoint = "https://api.x.ai/v1"
api_key = "${XAI_API_KEY}"
-[ollama]
-enabled = false
-endpoint = "http://localhost:11434/v1"
-
-[blackboxai]
-display_name = "BlackBox AI"
-enabled = false
-endpoint = "https://api.blackbox.ai/v1"
-api_key = "${BLACKBOX_API_KEY}"
-
-[fal]
-display_name = "FAL"
-enabled = false
-api_key = "${FAL_API_KEY}"
-
[internal] # software-only backend, runs internally, without AI
enabled = true
diff --git a/pipelex/kit/configs/inference/backends/anthropic.toml b/pipelex/kit/configs/inference/backends/anthropic.toml
index 4b57964c4..66721e2c1 100644
--- a/pipelex/kit/configs/inference/backends/anthropic.toml
+++ b/pipelex/kit/configs/inference/backends/anthropic.toml
@@ -89,3 +89,11 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+
+["claude-4.5-haiku"]
+model_id = "claude-haiku-4-5-20251001"
+max_tokens = 64000
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 100
+costs = { input = 1.0, output = 5.0 }
diff --git a/pipelex/kit/configs/inference/backends/azure_openai.toml b/pipelex/kit/configs/inference/backends/azure_openai.toml
index 135638ad5..930f161fd 100644
--- a/pipelex/kit/configs/inference/backends/azure_openai.toml
+++ b/pipelex/kit/configs/inference/backends/azure_openai.toml
@@ -80,6 +80,12 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.1, output = 4.4 }
+[o3]
+model_id = "o3-2025-04-16"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 2, output = 8 }
+
# --- GPT-5 Series -------------------------------------------------------------
[gpt-5-mini]
model_id = "gpt-5-mini-2025-08-07"
diff --git a/pipelex/kit/configs/inference/backends/bedrock.toml b/pipelex/kit/configs/inference/backends/bedrock.toml
index c27baf0b5..cea91344d 100644
--- a/pipelex/kit/configs/inference/backends/bedrock.toml
+++ b/pipelex/kit/configs/inference/backends/bedrock.toml
@@ -92,8 +92,6 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
-# anthropic.claude-sonnet-4-5-20250929-v1:0,Anthropic,arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0,
-
["claude-4.5-sonnet"]
sdk = "bedrock_anthropic"
model_id = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
@@ -102,3 +100,12 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+
+["claude-4.5-haiku"]
+sdk = "bedrock_anthropic"
+model_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
+max_tokens = 8192
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 100
+costs = { input = 1.0, output = 5.0 }
diff --git a/pipelex/kit/configs/inference/backends/pipelex_inference.toml b/pipelex/kit/configs/inference/backends/pipelex_inference.toml
index 805b761ad..e516e274e 100644
--- a/pipelex/kit/configs/inference/backends/pipelex_inference.toml
+++ b/pipelex/kit/configs/inference/backends/pipelex_inference.toml
@@ -103,6 +103,12 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 3, output = 15 }
+["claude-4.5-haiku"]
+model_id = "pipelex/claude-4.5-haiku"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+costs = { input = 1, output = 5 }
+
# --- Gemini LLMs --------------------------------------------------------------
["gemini-2.0-flash"]
model_id = "gemini/gemini-2.0-flash"
diff --git a/pipelex/kit/configs/inference/deck/base_deck.toml b/pipelex/kit/configs/inference/deck/base_deck.toml
index 3fcc50d11..12f516755 100644
--- a/pipelex/kit/configs/inference/deck/base_deck.toml
+++ b/pipelex/kit/configs/inference/deck/base_deck.toml
@@ -28,27 +28,59 @@ best-gemini = "gemini-2.5-pro"
best-mistral = "mistral-medium"
best-grok = "grok-3"
-cheap-gpt = "gpt-4o-mini"
-cheap_llm_for_text = "gpt-4o-mini"
-cheap_llm_for_object = "gpt-4o-mini"
-cheap_llm_for_vision = "gemini-2.5-flash-lite"
+# Image generation aliases
+base-img-gen = "flux-pro/v1.1"
+best-img-gen = "flux-pro/v1.1-ultra"
+fast-img-gen = "fast-lightning-sdxl"
-base_llm_for_text = "gpt-4o-mini"
-base_llm_for_object = "gpt-4o-mini"
+####################################################################################################
+# Waterfalls
+####################################################################################################
+[waterfalls]
smart_llm = [
"claude-4.5-sonnet",
"claude-4.1-opus",
"claude-4-sonnet",
"gemini-2.5-pro",
- "gpt-4o",
+ "gpt-4o", # we're using gpt-4o here rather than gpt-5 until we'll handle reasoning parameters
+ "grok-3",
+]
+smart_llm_with_vision = [
+ "claude-4.5-sonnet",
+ "claude-4.1-opus",
+ "claude-4-sonnet",
+ "gemini-2.5-pro",
+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters
+ "grok-3",
+]
+smart_llm_for_structured = [
+ "claude-4.5-sonnet",
+ "claude-4.1-opus",
+ "claude-4-sonnet",
+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters
]
-llm_for_large_codebase = ["gemini-2.5-pro", "claude-4.5-sonnet", "gpt-5"]
-# Image generation aliases
-base-img-gen = "flux-pro/v1.1"
-best-img-gen = "flux-pro/v1.1-ultra"
-fast-img-gen = "fast-lightning-sdxl"
+llm_for_large_codebase = [
+ "gemini-2.5-pro",
+ "claude-4.5-sonnet",
+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters
+ "gemini-2.5-flash",
+ "grok-3",
+]
+cheap_llm = [
+ "gpt-4o-mini",
+ "gemini-2.5-flash-lite",
+ "mistral-small",
+ "claude-3-haiku",
+ "grok-3-mini",
+]
+cheap_llm_for_vision = [
+ "gpt-4o-mini",
+ "gemini-2.5-flash-lite",
+ "claude-3-haiku",
+]
+cheap_llm_for_structured = ["gpt-4o-mini", "mistral-small", "claude-3-haiku"]
####################################################################################################
# LLM Presets
@@ -56,73 +88,47 @@ fast-img-gen = "fast-lightning-sdxl"
[llm.presets]
-####################################################################################################
-# LLM Presets — General purpose
-
-cheap_llm_for_text = { model = "cheap_llm_for_text", temperature = 0.5 }
-cheap_llm_for_short_text = { model = "cheap_llm_for_text", temperature = 0.5, max_tokens = 50 }
-cheap_llm_for_object = { model = "cheap_llm_for_object", temperature = 0.5 }
-cheap_llm_to_structure = { model = "cheap_llm_for_object", temperature = 0.1 }
-
-llm_for_testing_gen_text = { model = "base_llm_for_text", temperature = 0.5 }
-llm_for_testing_gen_object = { model = "base_llm_for_object", temperature = 0.5 }
-llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }
-llm_for_testing_vision_structured = { model = "cheap-gpt", temperature = 0.5 }
-
-####################################################################################################
-# LLM Presets — Specific skills
-
-# Cheap llm
-llm_cheap = { model = "base-gemini", temperature = 0.1 }
+# LLM Presets — Specific skills -------------------------------------------------------------
# Generation skills
llm_for_factual_writing = { model = "base-gpt", temperature = 0.1 }
llm_for_creative_writing = { model = "base-gpt", temperature = 0.9 }
-llm_to_write_questions = { model = "gpt-4o-mini", temperature = 0.3 }
-
-# Reasoning skills
-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }
-llm_to_reason_on_diagram = { model = "base-claude", temperature = 0.5 }
-llm_to_analyze_data = { model = "base-claude", temperature = 0.5 }
+llm_for_writing_cheap = { model = "gpt-4o-mini", temperature = 0.3 }
-# Search and answer skills
-llm_to_answer_easy_questions = { model = "gpt-4o-mini", temperature = 0.3 }
-llm_to_answer_hard_questions = { model = "base-claude", temperature = 0.3 }
+# Retrieve and answer questions skills
+llm_to_answer_questions_cheap = { model = "gpt-4o-mini", temperature = 0.3 }
+llm_to_answer_questions = { model = "base-claude", temperature = 0.3 }
llm_to_retrieve = { model = "base-claude", temperature = 0.1 }
-llm_for_enrichment = { model = "gpt-4o", temperature = 0.1 }
-llm_to_enrich = { model = "base-claude", temperature = 0.1 }
-llm_for_question_and_excerpt_reformulation = { model = "gpt-4o", temperature = 0.9 }
# Engineering skills
-llm_to_engineer = { model = "smart_llm", temperature = 0.2 }
-llm_to_write = { model = "base-gemini", temperature = 0.2 }
-llm_to_pipelex = { model = "base-claude", temperature = 0.2 }
+llm_to_engineer = { model = "smart_llm_for_structured", temperature = 0.2 }
llm_to_code = { model = "base-claude", temperature = 0.1 }
llm_to_analyze_large_codebase = { model = "base-claude", temperature = 0.1 }
-# Image skills
-llm_for_basic_vision = { model = "gpt-4o-mini", temperature = 0.1 }
-llm_to_write_img_gen_prompt = { model = "base-claude", temperature = 0.2 }
-llm_to_describe_img = { model = "base-claude", temperature = 0.5 }
-llm_for_visual_analysis = { model = "base-claude", temperature = 0.5 }
-llm_for_visual_design = { model = "base-claude", temperature = 0.5 }
-llm_to_design_fashion = { model = "base-claude", temperature = 0.7 }
+# Vision skills
+llm_for_img_to_text_cheap = { model = "gpt-4o-mini", temperature = 0.1 }
llm_for_img_to_text = { model = "base-claude", temperature = 0.1 }
+llm_for_diagram_to_text = { model = "base-claude", temperature = 0.3 }
+llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }
-# Extraction skills
-llm_to_extract_diagram = { model = "base-claude", temperature = 0.5 }
-llm_to_extract_invoice = { model = "claude-4.5-sonnet", temperature = 0.1 }
-llm_to_extract_invoice_from_scan = { model = "base-claude", temperature = 0.5 }
-llm_to_extract_legal_terms = { model = "base-claude", temperature = 0.1 }
-llm_to_extract_tables = { model = "base-claude", temperature = 0.1 }
+# Image generation prompting skills
+llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }
+llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }
+
+# LLM Presets — For Testing ---------------------------------------------------------------------
+
+llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }
+llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.5 }
+llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }
+llm_for_testing_vision_structured = { model = "cheap_llm_for_vision", temperature = 0.5 }
####################################################################################################
# LLM Choices
####################################################################################################
[llm.choice_defaults]
-for_text = "cheap_llm_for_text"
-for_object = "cheap_llm_for_object"
+for_text = "cheap_llm"
+for_object = "cheap_llm_for_structured"
####################################################################################################
# Extract Presets
@@ -144,7 +150,6 @@ choice_default = "gen_image_basic"
[img_gen.presets]
-
# General purpose
gen_image_basic = { model = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }
gen_image_fast = { model = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }
diff --git a/pipelex/kit/configs/inference/routing_profiles.toml b/pipelex/kit/configs/inference/routing_profiles.toml
index a5c5a0ce4..592533daf 100644
--- a/pipelex/kit/configs/inference/routing_profiles.toml
+++ b/pipelex/kit/configs/inference/routing_profiles.toml
@@ -1,12 +1,12 @@
# Routing profile library - Routes models to their backends
-# ================================================
+# =========================================================================================
# This file controls which backend serves which model.
# Simply change the 'active' field to switch profiles,
# or you can add your own custom profiles.
#
# Documentation: https://docs.pipelex.com
# Support: https://go.pipelex.com/discord
-# ================================================
+# =========================================================================================
# Which profile to use (change this to switch routing)
active = "pipelex_first"
@@ -21,17 +21,21 @@ active = "pipelex_first"
# 3. The .pipelex/inference/backends.toml is already configured with api_key = "${PIPELEX_INFERENCE_API_KEY}"
# which will get the key from the environment variable.
-# ============================================
+# =========================================================================================
# Routing Profiles
-# ============================================
+# =========================================================================================
[profiles.pipelex_first]
description = "Use Pipelex Inference backend for all its supported models"
default = "pipelex_inference"
-
-[profiles.all_blackboxai]
-description = "Use BlackBoxAI backend for all its supported models"
-default = "blackboxai"
+fallback_order = [
+ "pipelex_inference",
+ "azure_openai",
+ "bedrock",
+ "blackboxai",
+ "mistral",
+ "fal",
+]
[profiles.pipelex_first.routes]
# Pattern matching: "model-pattern" = "backend-name"
@@ -39,33 +43,82 @@ default = "blackboxai"
"claude-*" = "pipelex_inference"
"grok-*" = "pipelex_inference"
"gemini-*" = "pipelex_inference"
+
+[profiles.pipelex_first.optional_routes] # Each optional route is considered only if its backend is available
"*-sdxl" = "blackboxai"
"flux-*" = "blackboxai"
"nano-banana" = "blackboxai"
"gpt-image-1" = "openai"
"mistral-ocr" = "mistral"
+[profiles.all_anthropic]
+description = "Use Anthropic backend for all its supported models"
+default = "anthropic"
+
+[profiles.all_azure_openai]
+description = "Use Azure OpenAI backend for all its supported models"
+default = "azure_openai"
+
[profiles.all_bedrock]
description = "Use Bedrock backend for all its supported models"
default = "bedrock"
-[profiles.all_anthropic]
-description = "Use Anthropic backend for all its supported models"
-default = "anthropic"
+[profiles.all_blackboxai]
+description = "Use BlackBoxAI backend for all its supported models"
+default = "blackboxai"
+
+[profiles.all_fal]
+description = "Use FAL backend for all its supported models"
+default = "fal"
-[profiles.all_gemini]
+[profiles.all_google]
description = "Use Google GenAI backend for all its supported models"
default = "google"
-[profiles.all_azure_openai]
-description = "Use Azure OpenAI backend for all its supported models"
-default = "azure_openai"
+[profiles.all_mistral]
+description = "Use Mistral backend for all its supported models"
+default = "mistral"
+
+[profiles.all_ollama]
+description = "Use Ollama backend for all its supported models"
+default = "ollama"
+
+[profiles.all_openai]
+description = "Use OpenAI backend for all its supported models"
+default = "openai"
+
+[profiles.all_vertexai]
+description = "Use Vertex AI backend for all its supported models"
+default = "vertexai"
+
+[profiles.all_xai]
+description = "Use xAI backend for all its supported models"
+default = "xai"
+
+# =========================================================================================
+# Custom Profiles
+# =========================================================================================
+# Add your own profiles below following the same pattern:
+#
+# [profiles.your_profile_name]
+# description = "What this profile does"
+# default = "backend-name" # Where to route models by default
+# [profiles.your_profile_name.routes]
+# "model-pattern" = "backend-name" # Specific routing rules
+#
+# Pattern matching supports:
+# - Exact names: "gpt-4o-mini"
+# - Wildcards: "claude-*" (matches all models starting with claude-)
+# - Partial wildcards: "*-sonnet" (matches all sonnet variants)
-[profiles.custom_routing]
-description = "Custom routing"
+# =========================================================================================
+# Example of a custom routing profile with mostly pattern matching and one specific model
+# =========================================================================================
+[profiles.example_routing_using_patterns]
+description = "Example routing profile using patterns"
default = "pipelex_inference"
-[profiles.custom_routing.routes]
+[profiles.example_routing_using_patterns.routes]
# Pattern matching: "model-pattern" = "backend-name"
"gpt-*" = "azure_openai"
"claude-*" = "bedrock"
@@ -75,10 +128,14 @@ default = "pipelex_inference"
"flux-*" = "fal"
"gpt-image-1" = "openai"
-[profiles.to_test_various_backends]
-description = "Route models to various backends"
+# =========================================================================================
+# Example of a custom routing profile with specific model matching
+# =========================================================================================
+
+[profiles.example_routing_using_specific_models]
+description = "Example routing profile using specific models"
-[profiles.to_test_various_backends.routes]
+[profiles.example_routing_using_specific_models.routes]
"gpt-5-nano" = "pipelex_inference"
"gpt-4o-mini" = "blackboxai"
"gpt-5-mini" = "openai"
@@ -93,19 +150,3 @@ description = "Route models to various backends"
"grok-3" = "pipelex_inference"
"grok-3-mini" = "xai"
-
-# ============================================
-# Custom Profiles
-# ============================================
-# Add your own profiles below following the same pattern:
-#
-# [profiles.your_profile_name]
-# description = "What this profile does"
-# default = "backend-name" # Where to route models by default
-# [profiles.your_profile_name.routes]
-# "model-pattern" = "backend-name" # Specific routing rules
-#
-# Pattern matching supports:
-# - Exact names: "gpt-4o-mini"
-# - Wildcards: "claude-*" (matches all models starting with claude-)
-# - Partial wildcards: "*-sonnet" (matches all sonnet variants)
diff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.toml
index 570dae25e..a1faa165f 100644
--- a/pipelex/kit/configs/pipelex.toml
+++ b/pipelex/kit/configs/pipelex.toml
@@ -49,6 +49,13 @@ default_log_level = "INFO"
[pipelex.log_config.package_log_levels]
pipelex = "INFO"
+[cogt]
+[cogt.model_deck_config]
+# Uncomment to disable model fallback: it will raise errors instead of using secondary model options
+# is_model_fallback_enabled = false
+# Uncomment to change the reaction to missing presets: "raise" (default), "log" or "none"
+# missing_presets_reaction = "raise"
+
[cogt.llm_config]
# Uncomment any of these to enable dumping the inputs or outputs of text-genration with an LLM
# is_dump_text_prompts_enabled = true
diff --git a/pipelex/kit/cursor_export.py b/pipelex/kit/cursor_export.py
index ca207b8a9..ebc8b38d8 100644
--- a/pipelex/kit/cursor_export.py
+++ b/pipelex/kit/cursor_export.py
@@ -7,7 +7,7 @@ import typer
import yaml
from pipelex.kit.index_models import KitIndex
-from pipelex.kit.paths import get_agents_dir
+from pipelex.kit.paths import get_kit_agents_dir
def _iter_agent_files(agents_dir: Traversable) -> Iterable[tuple[str, str]]:
@@ -52,7 +52,7 @@ def export_cursor_rules(repo_root: Path, idx: KitIndex, dry_run: bool = False) -
idx: Kit index configuration
dry_run: If True, only print what would be done
"""
- agents_dir = get_agents_dir()
+ agents_dir = get_kit_agents_dir()
out_dir = repo_root / ".cursor" / "rules"
out_dir.mkdir(parents=True, exist_ok=True)
@@ -76,7 +76,7 @@ def remove_cursor_rules(repo_root: Path, dry_run: bool = False) -> None:
repo_root: Repository root directory
dry_run: If True, only print what would be done
"""
- agents_dir = get_agents_dir()
+ agents_dir = get_kit_agents_dir()
out_dir = repo_root / ".cursor" / "rules"
if not out_dir.exists():
diff --git a/pipelex/kit/migrations/migrate_0.11.0_0.12.x.md b/pipelex/kit/migrations/migrate_0.11.0_0.12.x.md
index 7959a4155..4877f2b6d 100644
--- a/pipelex/kit/migrations/migrate_0.11.0_0.12.x.md
+++ b/pipelex/kit/migrations/migrate_0.11.0_0.12.x.md
@@ -682,13 +682,13 @@ default_outcome = "fail"
**Before (.pipelex/inference/deck/base_deck.toml):**
```toml
[presets.llm]
-llm_for_complex_reasoning = { llm_handle = "claude-3-5-sonnet", temperature = 1 }
+llm_to_engineer = { llm_handle = "claude-3-5-sonnet", temperature = 1 }
```
**After:**
```toml
[presets.llm]
-llm_for_complex_reasoning = { model = "claude-3-5-sonnet", temperature = 1 }
+llm_to_engineer = { model = "claude-3-5-sonnet", temperature = 1 }
```
### Image generation presets
diff --git a/pipelex/kit/migrations_export.py b/pipelex/kit/migrations_export.py
index 792124310..a21a35df1 100644
--- a/pipelex/kit/migrations_export.py
+++ b/pipelex/kit/migrations_export.py
@@ -2,7 +2,7 @@ from pathlib import Path
import typer
-from pipelex.kit.paths import get_migrations_dir
+from pipelex.kit.paths import get_kit_migrations_dir
def export_migration_instructions(repo_root: Path, dry_run: bool = False) -> None:
@@ -12,7 +12,7 @@ def export_migration_instructions(repo_root: Path, dry_run: bool = False) -> Non
repo_root: Repository root directory
dry_run: If True, only print what would be done
"""
- migrations_dir = get_migrations_dir()
+ migrations_dir = get_kit_migrations_dir()
out_dir = repo_root / ".pipelex" / "migrations"
out_dir.mkdir(parents=True, exist_ok=True)
diff --git a/pipelex/kit/paths.py b/pipelex/kit/paths.py
index db49b0dbc..7374360d0 100644
--- a/pipelex/kit/paths.py
+++ b/pipelex/kit/paths.py
@@ -11,7 +11,7 @@ def get_kit_root() -> Traversable:
return files("pipelex.kit")
-def get_agents_dir() -> Traversable:
+def get_kit_agents_dir() -> Traversable:
"""Get the agents directory within the kit package.
Returns:
@@ -20,7 +20,7 @@ def get_agents_dir() -> Traversable:
return get_kit_root() / "agent_rules"
-def get_configs_dir() -> Traversable:
+def get_kit_configs_dir() -> Traversable:
"""Get the configs directory within the kit package.
Returns:
@@ -29,7 +29,7 @@ def get_configs_dir() -> Traversable:
return get_kit_root() / "configs"
-def get_migrations_dir() -> Traversable:
+def get_kit_migrations_dir() -> Traversable:
"""Get the migrations directory within the kit package.
Returns:
diff --git a/pipelex/kit/targets_update.py b/pipelex/kit/targets_update.py
index 3f80f530f..c4038b81f 100644
--- a/pipelex/kit/targets_update.py
+++ b/pipelex/kit/targets_update.py
@@ -7,7 +7,7 @@ import typer
from pipelex.kit.index_models import KitIndex, Target
from pipelex.kit.markers import find_span, replace_span, wrap
-from pipelex.kit.paths import get_agents_dir
+from pipelex.kit.paths import get_kit_agents_dir
def _read_agent_file(agents_dir: Traversable, name: str) -> str:
@@ -58,7 +58,7 @@ def build_merged_rules(idx: KitIndex, agent_set: str | None = None) -> str:
Returns:
Merged markdown content with demoted headings
"""
- agents_dir = get_agents_dir()
+ agents_dir = get_kit_agents_dir()
if agent_set is None:
agent_set = idx.agent_rules.default_set
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index cf1ff455d..dde79d4e4 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -340,14 +340,14 @@ class LibraryManager(LibraryManagerAbstract):
try:
blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()
except FileNotFoundError as file_not_found_error:
- msg = f"Could not find PLX blueprint at '{plx_file_path}'"
+ msg = f"Could not find PLX bundle at '{plx_file_path}'"
raise LibraryLoadingError(msg) from file_not_found_error
except PipeDefinitionError as pipe_def_error:
- msg = f"Could not load PLX blueprint from '{plx_file_path}': {pipe_def_error}"
+ msg = f"Could not load PLX bundle from '{plx_file_path}' because of: {pipe_def_error}"
raise LibraryLoadingError(msg) from pipe_def_error
except ValidationError as validation_error:
validation_error_msg = report_validation_error(category="plx", validation_error=validation_error)
- msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {validation_error_msg}"
+ msg = f"Could not load PLX bundle from '{plx_file_path}' because of: {validation_error_msg}"
raise LibraryLoadingError(msg) from validation_error
blueprint.source = str(plx_file_path)
blueprints.append(blueprint)
diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.py
index ff422d21a..820dc81b3 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch.py
@@ -50,7 +50,7 @@ class PipeBatch(PipeController):
return self
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
self._validate_required_variables()
def _validate_required_variables(self) -> Self:
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index d5a637dfb..515853bd9 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -222,7 +222,7 @@ class PipeCondition(PipeController):
raise extraneous_input_var_error
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
"""Perform full validation after all libraries are loaded.
This is called after all pipes and concepts are available.
"""
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.py
index 060e4777d..784e4e2f2 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py
@@ -162,7 +162,7 @@ class PipeParallel(PipeController):
raise extraneous_input_var_error
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
"""Perform full validation after all libraries are loaded.
This is called after all pipes and concepts are available.
"""
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.py
index abeb6ee5b..075954f0b 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py
@@ -168,7 +168,7 @@ of sequence pipe '{self.code}' is not compatible with the output concept '{self.
raise extraneous_input_var_error
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
"""Perform full validation after all libraries are loaded.
This is called after all pipes and concepts are available.
"""
diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.py
index 67675a2cd..8cda93e4f 100644
--- a/pipelex/pipe_operators/compose/pipe_compose.py
+++ b/pipelex/pipe_operators/compose/pipe_compose.py
@@ -77,7 +77,7 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
pass
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
pass
@override
diff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.py
index e32276a51..da32dd25d 100644
--- a/pipelex/pipe_operators/extract/pipe_extract.py
+++ b/pipelex/pipe_operators/extract/pipe_extract.py
@@ -61,7 +61,7 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):
return self
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
self._validate_inputs()
if self.extract_choice:
check_extract_choice_with_deck(extract_choice=self.extract_choice)
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
index 4d2fcfa06..00447130d 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
@@ -98,7 +98,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
return self
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
self._validate_inputs()
if self.img_gen:
check_img_gen_choice_with_deck(img_gen_choice=self.img_gen)
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index 5c63350a3..e8e8faaf2 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -80,11 +80,11 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
return self
@override
- def validate_with_libraries(self):
+ def _validate_with_libraries(self):
self.validate_inputs()
self.llm_prompt_spec.validate_with_libraries()
if self.llm_choices:
- for llm_choice in self.llm_choices.list_choices():
+ for llm_choice in self.llm_choices.list_choice_strings():
check_llm_choice_with_deck(llm_choice=llm_choice)
@override
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index dec6d33fb..67aa04255 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -4,10 +4,12 @@ from typing import Generic, Literal, TypeVar
from typing_extensions import override
from pipelex import log, pretty_print_md
+from pipelex.cogt.exceptions import ModelNotFoundError, ModelWaterfallError
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.stuffs.text_content import TextContent
+from pipelex.exceptions import PipeOperatorModelAvailabilityError
from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
@@ -39,41 +41,61 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
)
job_metadata.update(updated_metadata=updated_metadata)
- match pipe_run_params.run_mode:
- case PipeRunMode.LIVE:
- if self.class_name not in ["PipeCompose", "PipeLLMPrompt"]:
- name = f"Running [cyan]{self.class_name}[/cyan]"
+ try:
+ match pipe_run_params.run_mode:
+ case PipeRunMode.LIVE:
+ if self.class_name not in ["PipeCompose", "PipeLLMPrompt"]:
+ name = f"Running [cyan]{self.class_name}[/cyan]"
+ indent_level = len(pipe_run_params.pipe_stack) - 1
+ indent = " " * indent_level
+ label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name} → [green]{self.code}[/green]"
+ log.info(f"{label} → [red]{self.output.code}[/red]")
+ pipe_output = await self._run_operator_pipe(
+ job_metadata=job_metadata,
+ working_memory=working_memory,
+ pipe_run_params=pipe_run_params,
+ output_name=output_name,
+ )
+ if isinstance(pipe_output.main_stuff.content, TextContent):
+ print()
+ pretty_print_md(pipe_output.main_stuff_as_str, title=f"PipeOutput of pipe {self.code}")
+ print()
+ else:
+ print()
+ pipe_output.main_stuff.pretty_print_stuff(title=f"PipeOutput of pipe {self.code}: {self.output.code}")
+ print()
+ case PipeRunMode.DRY:
+ name = f"Dry run [cyan]{self.class_name}[/cyan]"
indent_level = len(pipe_run_params.pipe_stack) - 1
indent = " " * indent_level
- label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name} → [green]{self.code}[/green]"
+ label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name}: [green]{self.code}[/green]"
log.info(f"{label} → [red]{self.output.code}[/red]")
- pipe_output = await self._run_operator_pipe(
- job_metadata=job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- output_name=output_name,
- )
- if isinstance(pipe_output.main_stuff.content, TextContent):
- print()
- pretty_print_md(pipe_output.main_stuff_as_str, title=f"PipeOutput of pipe {self.code}")
- print()
- else:
- print()
- pipe_output.main_stuff.pretty_print_stuff(title=f"PipeOutput of pipe {self.code}: {self.output.code}")
- print()
- case PipeRunMode.DRY:
- name = f"Dry run [cyan]{self.class_name}[/cyan]"
- indent_level = len(pipe_run_params.pipe_stack) - 1
- indent = " " * indent_level
- label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name}: [green]{self.code}[/green]"
- log.info(f"{label} → [red]{self.output.code}[/red]")
- pipe_output = await self._dry_run_operator_pipe(
- job_metadata=job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- output_name=output_name,
- )
-
+ pipe_output = await self._dry_run_operator_pipe(
+ job_metadata=job_metadata,
+ working_memory=working_memory,
+ pipe_run_params=pipe_run_params,
+ output_name=output_name,
+ )
+ except ModelWaterfallError as model_waterfall_error:
+ raise PipeOperatorModelAvailabilityError(
+ message=model_waterfall_error.message,
+ run_mode=pipe_run_params.run_mode,
+ pipe_type=self.class_name,
+ pipe_code=self.code,
+ pipe_stack=pipe_run_params.pipe_stack,
+ model_handle=model_waterfall_error.model_handle,
+ fallback_list=model_waterfall_error.fallback_list,
+ ) from model_waterfall_error
+ except ModelNotFoundError as model_not_found_error:
+ raise PipeOperatorModelAvailabilityError(
+ message=model_not_found_error.message,
+ run_mode=pipe_run_params.run_mode,
+ pipe_type=self.class_name,
+ pipe_code=self.code,
+ pipe_stack=pipe_run_params.pipe_stack,
+ model_handle=model_not_found_error.model_handle,
+ fallback_list=None,
+ ) from model_not_found_error
pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)
return pipe_output
diff --git a/pipelex/pipe_run/dry_run.py b/pipelex/pipe_run/dry_run.py
index d9459627b..e0c0971aa 100644
--- a/pipelex/pipe_run/dry_run.py
+++ b/pipelex/pipe_run/dry_run.py
@@ -100,13 +100,28 @@ async def dry_run_pipes(pipes: list[PipeAbstract], run_in_parallel: bool = True,
"""Parallel execution using ThreadPoolExecutor"""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
- return loop.run_until_complete(dry_run_pipe(pipe, raise_on_failure=raise_on_failure))
+ try:
+ return loop.run_until_complete(dry_run_pipe(pipe, raise_on_failure=raise_on_failure))
+ finally:
+ loop.close()
with ThreadPoolExecutor() as executor:
futures = [asyncio.get_running_loop().run_in_executor(executor, functools.partial(run_pipe_in_thread, pipe)) for pipe in pipes]
for future in asyncio.as_completed(futures):
- output = await future
- results[output.pipe_code] = output
+ try:
+ output = await future
+ results[output.pipe_code] = output
+ except Exception as exc:
+ # If raise_on_failure is True, re-raise the first exception encountered
+ # Otherwise, this shouldn't happen as dry_run_pipe should return a DryRunOutput
+ if raise_on_failure:
+ # Cancel remaining futures
+ for f in futures:
+ if not f.done():
+ f.cancel()
+ raise
+ # This path shouldn't normally be reached, but handle it gracefully
+ log.error(f"Unexpected exception in dry run: {exc}")
else:
for pipe in pipes:
results[pipe.code] = await dry_run_pipe(pipe, raise_on_failure=raise_on_failure)
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 705e04cd2..f5145b725 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -18,13 +18,13 @@ from pipelex.cogt.exceptions import (
InferenceBackendLibraryValidationError,
ModelDeckNotFoundError,
ModelDeckValidationError,
+ RoutingProfileDisabledBackendError,
RoutingProfileLibraryNotFoundError,
- RoutingProfileValidationError,
)
from pipelex.cogt.inference.inference_manager import InferenceManager
from pipelex.cogt.models.model_manager import ModelManager
from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract
-from pipelex.config import PipelexConfig, get_config
+from pipelex.config import ConfigPaths, PipelexConfig, get_config
from pipelex.core.concepts.concept_library import ConceptLibrary
from pipelex.core.domains.domain_library import DomainLibrary
from pipelex.core.pipes.pipe_library import PipeLibrary
@@ -53,7 +53,7 @@ from pipelex.system.environment import get_optional_env
from pipelex.system.registries.func_registry import func_registry
from pipelex.system.runtime import IntegrationMode, runtime_manager
from pipelex.system.telemetry.observer_telemetry import ObserverTelemetry
-from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryConfig
+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryConfig, TelemetryMode
from pipelex.system.telemetry.telemetry_manager import DO_NOT_TRACK_ENV_VAR_KEY, TelemetryManager
from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract, TelemetryManagerNoOp
from pipelex.test_extras.registry_test_models import TestRegistryModels
@@ -71,10 +71,10 @@ PACKAGE_NAME, PACKAGE_VERSION = get_package_info()
class Pipelex(metaclass=MetaSingleton):
def __init__(
self,
- config_dir_path: str = "./pipelex",
+ config_dir_path: str | None = None,
config_cls: type[ConfigRoot] | None = None,
) -> None:
- self.config_dir_path = config_dir_path
+ self.config_dir_path = config_dir_path or ConfigPaths.DEFAULT_CONFIG_DIR_PATH
self.pipelex_hub = PipelexHub()
set_pipelex_hub(self.pipelex_hub)
@@ -119,7 +119,7 @@ class Pipelex(metaclass=MetaSingleton):
log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")
@staticmethod
- def _get_config_not_found_error_msg(component_name: str) -> str:
+ def _get_config_file_not_found_error_msg(component_name: str) -> str:
"""Generate error message for missing config files."""
return f"Config files are missing for the {component_name}. Run `pipelex init config` to generate the missing files."
@@ -129,10 +129,10 @@ class Pipelex(metaclass=MetaSingleton):
msg = ""
cause_exc = validation_exc.__cause__
if cause_exc is None:
- msg += f"\nUnxpexted cause:{cause_exc}"
+ msg += f"\nUnexpexted error:{cause_exc}"
raise PipelexSetupError(msg) from cause_exc
if not isinstance(cause_exc, ValidationError):
- msg += f"\nUnxpexted cause:{cause_exc}"
+ msg += f"\nUnexpexted cause:{cause_exc}"
raise PipelexSetupError(msg) from cause_exc
report = report_validation_error(category="config", validation_error=cause_exc)
return f"""{msg}
@@ -170,7 +170,8 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
self.class_registry = class_registry or ClassRegistry()
self.pipelex_hub.set_class_registry(self.class_registry)
self.kajson_manager = KajsonManager(class_registry=self.class_registry)
- self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())
+ secrets_provider = secrets_provider or EnvSecretsProvider()
+ self.pipelex_hub.set_secrets_provider(secrets_provider=secrets_provider)
self.pipelex_hub.set_storage_provider(storage_provider)
# cogt
@@ -180,19 +181,20 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
self.pipelex_hub.set_models_manager(models_manager=self.models_manager)
try:
- self.models_manager.setup()
+ self.models_manager.setup(secrets_provider=secrets_provider)
except RoutingProfileLibraryNotFoundError as routing_not_found_exc:
- msg = self._get_config_not_found_error_msg("routing profile library")
+ msg = self._get_config_file_not_found_error_msg("routing profile library")
raise PipelexSetupError(msg) from routing_not_found_exc
except InferenceBackendLibraryNotFoundError as backend_not_found_exc:
- msg = self._get_config_not_found_error_msg("inference backend library")
+ msg = self._get_config_file_not_found_error_msg("inference backend library")
raise PipelexSetupError(msg) from backend_not_found_exc
except ModelDeckNotFoundError as deck_not_found_exc:
- msg = self._get_config_not_found_error_msg("model deck")
+ msg = self._get_config_file_not_found_error_msg("model deck")
raise PipelexSetupError(msg) from deck_not_found_exc
- except RoutingProfileValidationError as routing_validation_exc:
- msg = self._get_validation_error_msg("routing profile library", routing_validation_exc)
- raise PipelexSetupError(msg) from routing_validation_exc
+ except RoutingProfileDisabledBackendError as routing_profile_exc:
+ msg = f"Some backend(s) required for a routing profile is not enabled: {routing_profile_exc}"
+ raise PipelexSetupError(msg) from routing_profile_exc
+
except InferenceBackendLibraryValidationError as backend_validation_exc:
msg = self._get_validation_error_msg("inference backend library", backend_validation_exc)
raise PipelexSetupError(msg) from backend_validation_exc
@@ -200,6 +202,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
msg = self._get_validation_error_msg("model deck", deck_validation_exc)
msg += "\n\nIf you added your own config files to the model deck then you'll have to change them manually."
raise PipelexSetupError(msg) from deck_validation_exc
+
except InferenceBackendCredentialsError as credentials_exc:
backend_name = credentials_exc.backend_name
var_name = credentials_exc.key_name
@@ -257,11 +260,16 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
telemetry_config_toml = load_toml_from_path(path=config_path)
telemetry_config = TelemetryConfig.model_validate(telemetry_config_toml)
- if telemetry_config.respect_dnt and (dnt := get_optional_env(DO_NOT_TRACK_ENV_VAR_KEY)) and dnt.lower() not in ["false", "0"]:
- self.telemetry_manager = TelemetryManagerNoOp()
- log.debug(f"Telemetry is disabled by env var 'DO_NOT_TRACK' which is set to {dnt}")
- else:
- self.telemetry_manager = telemetry_manager or TelemetryManager(telemetry_config=telemetry_config)
+ match telemetry_config.telemetry_mode:
+ case TelemetryMode.OFF:
+ self.telemetry_manager = TelemetryManagerNoOp()
+ log.debug("Telemetry is disabled because telemetry_mode is set to 'off'")
+ case TelemetryMode.ANONYMOUS | TelemetryMode.IDENTIFIED:
+ if telemetry_config.respect_dnt and (dnt := get_optional_env(DO_NOT_TRACK_ENV_VAR_KEY)) and dnt.lower() not in ["false", "0"]:
+ self.telemetry_manager = TelemetryManagerNoOp()
+ log.debug(f"Telemetry is disabled by env var 'DO_NOT_TRACK' which is set to {dnt}")
+ else:
+ self.telemetry_manager = telemetry_manager or TelemetryManager(telemetry_config=telemetry_config)
else:
self.telemetry_manager = TelemetryManagerNoOp()
log.verbose(f"Telemetry is disabled because the integration mode '{integration_mode}' does not allow it")
@@ -399,6 +407,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
**kwargs,
)
pipelex_instance.setup_libraries()
+ pipelex_instance.models_manager.validate_model_deck()
log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} ready")
return pipelex_instance
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index b24454d64..ed8b85fb2 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -93,13 +93,9 @@ api_key_method = "env"
[cogt]
-[cogt.inference_config]
-inference_config_path = ".pipelex/inference"
-
-[cogt.inference_manager_config]
-is_auto_setup_preset_llm = true
-is_auto_setup_preset_img_gen = true
-is_auto_setup_preset_extract = true
+[cogt.model_deck_config]
+is_model_fallback_enabled = true
+missing_presets_reaction = "log"
[cogt.llm_config]
default_max_images = 100
diff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.py
index f662192ba..9c7d18981 100644
--- a/pipelex/plugins/openai/openai_llm_worker.py
+++ b/pipelex/plugins/openai/openai_llm_worker.py
@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any
import instructor
import openai
from instructor.exceptions import InstructorRetryException
-from openai import NOT_GIVEN, APIConnectionError, BadRequestError, NotFoundError
+from openai import NOT_GIVEN, APIConnectionError, BadRequestError, NotFoundError, omit
if TYPE_CHECKING:
from openai.types.chat import ChatCompletionMessage
@@ -84,7 +84,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
response = await self.openai_client_for_text.chat.completions.create(
model=self.inference_model.model_id,
temperature=temperature,
- max_tokens=llm_job.job_params.max_tokens or None,
+ max_tokens=llm_job.job_params.max_tokens or omit,
seed=llm_job.job_params.seed,
messages=messages,
)
diff --git a/pipelex/system/configuration/config_loader.py b/pipelex/system/configuration/config_loader.py
index 5b009aac5..0b524df54 100644
--- a/pipelex/system/configuration/config_loader.py
+++ b/pipelex/system/configuration/config_loader.py
@@ -1,4 +1,3 @@
-import importlib
import os
from typing import Any
@@ -72,39 +71,7 @@ class ConfigLoader:
config_path = os.path.join(self.pipelex_config_dir, CONFIG_NAME)
return load_toml_from_path_if_exists(config_path) or {}
- def load_inheritance_config(self, the_pipelex_config: dict[str, Any]):
- """Load the config by inheritance in a pyproject.toml file.
- This will be removed in the future.
- Requires to have a pyproject.toml file in the project root.
- """
- pyproject_path = os.path.join(os.getcwd(), "pyproject.toml")
- if not os.path.exists(pyproject_path):
- print(f"pyproject.toml not found in {os.getcwd()}")
- return
-
- def _find_package_path(package_name: str) -> str | None:
- """Find package path by importing it"""
- try:
- module = importlib.import_module(package_name)
- if hasattr(module, "__file__") and module.__file__:
- return os.path.dirname(module.__file__)
- except ImportError:
- pass
-
- return None
-
- pyproject = load_toml_from_path(path=pyproject_path)
- if "tool" in pyproject and "pipelex" in pyproject["tool"] and "config_inheritance" in pyproject["tool"]["pipelex"]:
- for config_name in pyproject["tool"]["pipelex"]["config_inheritance"]:
- package_path = _find_package_path(config_name)
- if package_path:
- config_path = os.path.join(package_path, "pipelex.toml")
- if os.path.exists(config_path):
- config = load_toml_from_path_if_exists(config_path)
- if config:
- deep_update(the_pipelex_config, config)
-
- def load_config(self, specific_config_path: str | None = None) -> dict[str, Any]:
+ def load_config(self) -> dict[str, Any]:
"""Load and merge configurations from pipelex and local config files.
The configuration is loaded and merged in the following order:
@@ -119,20 +86,16 @@ class ConfigLoader:
Dict[str, Any]: The merged configuration dictionary
"""
- #################### 1. Load pipelex config ####################
+ #################### 1. Load pipelex base config ####################
pipelex_config = self.get_pipelex_config()
- #################### 2. Load inheritance config for internal use ####################
- # TODO: Undocumented feature, soon to be removed.
- self.load_inheritance_config(pipelex_config)
-
- #################### 3. Load local (current project) config ####################
+ #################### 2. Load local (current project) config ####################
if not self.is_in_pipelex_config:
local_config = self.get_local_config()
if local_config:
deep_update(pipelex_config, local_config)
- #################### 4. Load overrides for the current project ####################
+ #################### 3. Load overrides for the current project ####################
list_of_overrides = [
*CONFIG_BASE_OVERRIDES_BEFORE_ENV,
runtime_manager.environment,
@@ -148,15 +111,6 @@ class ConfigLoader:
if override_dict := load_toml_from_path_if_exists(override_path):
deep_update(pipelex_config, override_dict)
- #################### 5. Load specific config ####################
- if specific_config_path:
- config = load_toml_from_path_if_exists(specific_config_path)
- if config:
- deep_update(pipelex_config, config)
- else:
- msg = f"Failed to load specific config from {specific_config_path}"
- raise ConfigError(msg)
-
return pipelex_config
diff --git a/pipelex/system/telemetry/telemetry_manager.py b/pipelex/system/telemetry/telemetry_manager.py
index 59c83a5eb..32bfefbf3 100644
--- a/pipelex/system/telemetry/telemetry_manager.py
+++ b/pipelex/system/telemetry/telemetry_manager.py
@@ -1,4 +1,5 @@
-from typing import Any, Callable
+from contextlib import contextmanager
+from typing import Any, Callable, Generator
import posthog
from posthog import Posthog, new_context, tag
@@ -23,7 +24,7 @@ class TelemetryManager(TelemetryManagerAbstract):
self.telemetry_config = telemetry_config
# Create PostHog client
- self.posthog = Posthog(
+ self.posthog_client = Posthog(
project_api_key=self.telemetry_config.project_api_key,
host=self.telemetry_config.host,
disable_geoip=not self.telemetry_config.geoip_enabled,
@@ -32,13 +33,13 @@ class TelemetryManager(TelemetryManagerAbstract):
)
# Store original capture_exception method
- self._original_capture_exception: Callable[..., Any] = self.posthog.capture_exception
+ self._original_capture_exception: Callable[..., Any] = self.posthog_client.capture_exception
# Wrap capture_exception to sanitize before sending
self._wrap_capture_exception()
posthog.privacy_mode = True
- posthog.default_client = self.posthog
+ posthog.default_client = self.posthog_client
def _handle_transmission_error(self, error: Exception | None, _items: list[dict[str, Any]]) -> None:
"""Handle errors that occur during telemetry transmission.
@@ -82,7 +83,7 @@ class TelemetryManager(TelemetryManagerAbstract):
return self._original_capture_exception(exception, **kwargs)
# Replace the method
- self.posthog.capture_exception = sanitized_capture_exception # type: ignore[method-assign]
+ self.posthog_client.capture_exception = sanitized_capture_exception # type: ignore[method-assign]
@override
def setup(self, integration_mode: IntegrationMode):
@@ -92,7 +93,7 @@ class TelemetryManager(TelemetryManagerAbstract):
tag(name=EventProperty.INTEGRATION, value=integration_mode)
tag(name=EventProperty.PIPELEX_VERSION, value=package_version)
tag(name=EventProperty.SETTING, value=Setting.TELEMETRY_MODE)
- self.posthog.capture(
+ self.posthog_client.capture(
EventName.TELEMETRY_JUST_ENABLED,
properties={
EventProperty.TELEMETRY_MODE: telemetry_mode,
@@ -102,7 +103,14 @@ class TelemetryManager(TelemetryManagerAbstract):
@override
def teardown(self):
- pass
+ if self.posthog_client:
+ try:
+ # PostHog client has a shutdown method to flush pending events
+ # and close background threads
+ self.posthog_client.shutdown()
+ except Exception as exc:
+ # Suppress any shutdown errors to avoid cascading failures
+ log.debug(f"Error during PostHog shutdown: {exc}")
@override
def track_event(self, event_name: EventName, properties: dict[EventProperty, Any] | None = None):
@@ -127,7 +135,7 @@ class TelemetryManager(TelemetryManagerAbstract):
log.verbose(f"Telemetry is off, skipping event '{event_name}'")
def _track_anonymous_event(self, event_name: str, properties: dict[str, Any]):
- if not self.posthog:
+ if not self.posthog_client:
return
if self.telemetry_config.dry_mode_enabled:
if properties:
@@ -136,11 +144,11 @@ class TelemetryManager(TelemetryManagerAbstract):
log.debug(f"Tracking anonymous event '{event_name}'. No properties.")
else:
properties["$process_person_profile"] = False
- self.posthog.capture(event_name, properties=properties)
+ self.posthog_client.capture(event_name, properties=properties)
log.verbose(f"Tracked anonymous event '{event_name}' with properties: {properties}")
def _track_identified_event(self, event_name: str, properties: dict[str, Any], user_id: str):
- if not self.posthog:
+ if not self.posthog_client:
return
if self.telemetry_config.dry_mode_enabled:
if properties:
@@ -148,5 +156,12 @@ class TelemetryManager(TelemetryManagerAbstract):
else:
log.debug(f"Tracking identified event '{event_name}'. No properties.")
else:
- self.posthog.capture(event_name, distinct_id=user_id, properties=properties)
+ self.posthog_client.capture(event_name, distinct_id=user_id, properties=properties)
log.verbose(f"Tracked identified event '{event_name}' with properties: {properties}")
+
+ @override
+ @contextmanager
+ def telemetry_context(self) -> Generator[None, None, None]:
+ """Context manager that uses PostHog's new_context when telemetry is enabled."""
+ with new_context():
+ yield
diff --git a/pipelex/system/telemetry/telemetry_manager_abstract.py b/pipelex/system/telemetry/telemetry_manager_abstract.py
index 529294468..b57403c88 100644
--- a/pipelex/system/telemetry/telemetry_manager_abstract.py
+++ b/pipelex/system/telemetry/telemetry_manager_abstract.py
@@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
-from typing import Any
+from contextlib import contextmanager
+from typing import Any, Generator
from typing_extensions import override
@@ -30,6 +31,11 @@ class TelemetryManagerAbstract(ABC):
def track_event(self, event_name: EventName, properties: dict[EventProperty, Any] | None = None):
pass
+ @abstractmethod
+ @contextmanager
+ def telemetry_context(self) -> Generator[None, None, None]:
+ """Safe context manager for telemetry that works whether telemetry is enabled or not."""
+
class TelemetryManagerNoOp(TelemetryManagerAbstract):
@override
@@ -43,3 +49,9 @@ class TelemetryManagerNoOp(TelemetryManagerAbstract):
@override
def track_event(self, event_name: EventName, properties: dict[EventProperty, Any] | None = None):
pass
+
+ @override
+ @contextmanager
+ def telemetry_context(self) -> Generator[None, None, None]:
+ """No-op context manager that doesn't use PostHog."""
+ yield
diff --git a/pipelex/tools/secrets/secrets_utils.py b/pipelex/tools/secrets/secrets_utils.py
index 11d9ea8c8..9d5d4972a 100644
--- a/pipelex/tools/secrets/secrets_utils.py
+++ b/pipelex/tools/secrets/secrets_utils.py
@@ -1,9 +1,9 @@
import re
-from pipelex.hub import get_secrets_provider
from pipelex.system.environment import EnvVarNotFoundError, get_optional_env, get_required_env
from pipelex.system.exceptions import ToolException
from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
from pipelex.types import StrEnum
@@ -32,7 +32,7 @@ class VarPrefix(StrEnum):
SECRET = "secret"
-def substitute_vars(content: str) -> str:
+def substitute_vars(content: str, secrets_provider: SecretsProviderAbstract) -> str:
"""Substitute variable placeholders with values from environment variables or secrets.
Supports the following placeholder formats:
@@ -43,6 +43,7 @@ def substitute_vars(content: str) -> str:
Args:
content: Text content with variable placeholders
+ secrets_provider: The secrets provider to use for secret lookups
Returns:
Content with variables substituted
@@ -57,7 +58,7 @@ def substitute_vars(content: str) -> str:
# Check if it's a fallback pattern (contains |)
if "|" in var_spec:
- return _handle_fallback_pattern(var_spec)
+ return _handle_fallback_pattern(var_spec, secrets_provider)
# Check if it has a prefix (env: or secret:)
if ":" in var_spec:
@@ -77,10 +78,10 @@ def substitute_vars(content: str) -> str:
case VarPrefix.ENV:
return _get_env_var(var_name)
case VarPrefix.SECRET:
- return _get_secret(var_name)
+ return _get_secret(var_name, secrets_provider)
else:
# Default behavior: use secrets provider
- return _get_secret(var_spec)
+ return _get_secret(var_spec, secrets_provider)
# Pattern matches ${VAR_NAME} or ${prefix:VAR_NAME} or ${env:VAR|secret:VAR}
# Restrict to not match across newlines, quotes, or nested braces
@@ -88,7 +89,7 @@ def substitute_vars(content: str) -> str:
return re.sub(pattern, replace_var, content)
-def _handle_fallback_pattern(var_spec: str) -> str:
+def _handle_fallback_pattern(var_spec: str, secrets_provider: SecretsProviderAbstract) -> str:
"""Handle fallback pattern like 'env:VAR|secret:VAR'."""
parts = [part.strip() for part in var_spec.split("|")]
@@ -113,13 +114,13 @@ def _handle_fallback_pattern(var_spec: str) -> str:
return value
case VarPrefix.SECRET:
try:
- return get_secrets_provider().get_secret(secret_id=var_name)
+ return secrets_provider.get_secret(secret_id=var_name)
except SecretNotFoundError:
continue # Try next option
else:
# No prefix, try as secret
try:
- return get_secrets_provider().get_secret(secret_id=part)
+ return secrets_provider.get_secret(secret_id=part)
except SecretNotFoundError:
continue # Try next option
msg = f"Could not get variable from fallback pattern: {var_spec}"
@@ -135,10 +136,10 @@ def _get_env_var(var_name: str) -> str:
raise VarNotFoundError(message=msg, var_name=var_name) from exc
-def _get_secret(secret_name: str) -> str:
+def _get_secret(secret_name: str, secrets_provider: SecretsProviderAbstract) -> str:
"""Get secret, raising VarNotFoundError if not found."""
try:
- return get_secrets_provider().get_secret(secret_id=secret_name)
+ return secrets_provider.get_secret(secret_id=secret_name)
except SecretNotFoundError as exc:
msg = f"Could not get variable '{secret_name}': {exc!s}"
raise VarNotFoundError(message=msg, var_name=secret_name) from exc
diff --git a/pipelex/tools/typing/pydantic_utils.py b/pipelex/tools/typing/pydantic_utils.py
index 9cc4f9705..a7688dd22 100644
--- a/pipelex/tools/typing/pydantic_utils.py
+++ b/pipelex/tools/typing/pydantic_utils.py
@@ -45,12 +45,12 @@ def analyze_pydantic_validation_error(exc: ValidationError) -> PydanticValidatio
error_msg = "Validation error(s):"
# Collect different types of validation errors
- missing_fields = [f"{'.'.join(map(str, err['loc']))}" for err in exc.errors() if err["type"] == "missing"]
- extra_fields = [f"{'.'.join(map(str, err['loc']))}: {err['input']}" for err in exc.errors() if err["type"] == "extra_forbidden"]
- type_errors = [f"{'.'.join(map(str, err['loc']))}: expected {err['type']}" for err in exc.errors() if err["type"] == "type_error"]
- value_errors = [f"{'.'.join(map(str, err['loc']))}: {err['msg']}" for err in exc.errors() if err["type"] == "value_error"]
+ missing_fields = [f"'{'.'.join(map(str, err['loc']))}'" for err in exc.errors() if err["type"] == "missing"]
+ extra_fields = [f"'{'.'.join(map(str, err['loc']))}'" for err in exc.errors() if err["type"] == "extra_forbidden"]
+ type_errors = [f"'{'.'.join(map(str, err['loc']))}': expected {err['type']}" for err in exc.errors() if err["type"] == "type_error"]
+ value_errors = [f"'{'.'.join(map(str, err['loc']))}': {err['msg']}" for err in exc.errors() if err["type"] == "value_error"]
enum_errors = [
- f"{'.'.join(map(str, err['loc']))}: invalid enum value '{err.get('input', 'unknown')}'" for err in exc.errors() if err["type"] == "enum"
+ f"'{'.'.join(map(str, err['loc']))}': invalid enum value '{err.get('input', 'unknown')}'" for err in exc.errors() if err["type"] == "enum"
]
union_tag_errors: list[str] = []
for err in exc.errors():
@@ -58,7 +58,7 @@ def analyze_pydantic_validation_error(exc: ValidationError) -> PydanticValidatio
field_path = ".".join(map(str, err["loc"]))
# Extract discriminator field name from context
discriminator = err.get("ctx", {}).get("discriminator", "type")
- union_tag_errors.append(f"{field_path}: missing required discriminator field '{discriminator}'")
+ union_tag_errors.append(f"'{field_path}': missing required discriminator field '{discriminator}'")
model_type_errors: list[str] = []
for err in exc.errors():
@@ -68,23 +68,23 @@ def analyze_pydantic_validation_error(exc: ValidationError) -> PydanticValidatio
expected_type = err.get("ctx", {}).get("class_name", "unknown model type")
actual_input = err.get("input", "unknown")
actual_type = type(actual_input).__name__ if actual_input != "unknown" else "unknown"
- model_type_errors.append(f"{field_path}: expected {expected_type}, got {actual_type}")
+ model_type_errors.append(f"'{field_path}': expected {expected_type}, got {actual_type}")
# Add each type of error to the message if present
if missing_fields:
- error_msg += f"\n\nMissing required fields: {missing_fields}"
+ error_msg += f"\n\nMissing required fields: {', '.join(missing_fields)}"
if extra_fields:
- error_msg += f"\n\nExtra forbidden fields: {extra_fields}"
+ error_msg += f"\n\nExtra forbidden fields: {', '.join(extra_fields)}"
if type_errors:
- error_msg += f"\n\nType errors: {type_errors}"
+ error_msg += f"\n\nType errors: {', '.join(type_errors)}"
if value_errors:
- error_msg += f"\n\nValue errors: {value_errors}"
+ error_msg += f"\n\nValue errors: {', '.join(value_errors)}"
if enum_errors:
- error_msg += f"\n\nEnum errors: {enum_errors}"
+ error_msg += f"\n\nEnum errors: {', '.join(enum_errors)}"
if union_tag_errors:
- error_msg += f"\n\nUnion discriminator errors: {union_tag_errors}"
+ error_msg += f"\n\nUnion discriminator errors: {', '.join(union_tag_errors)}"
if model_type_errors:
- error_msg += f"\n\nModel type errors: {model_type_errors}"
+ error_msg += f"\n\nModel type errors: {', '.join(model_type_errors)}"
# If none of the specific error types were found, add the raw error messages
if not any([missing_fields, extra_fields, type_errors, value_errors, enum_errors, union_tag_errors, model_type_errors]):
diff --git a/pipelex/urls.py b/pipelex/urls.py
index 7d90529a1..67eef86f5 100644
--- a/pipelex/urls.py
+++ b/pipelex/urls.py
@@ -5,3 +5,4 @@ class URLs:
changelog = "https://docs.pipelex.com/changelog/"
discord = "https://go.pipelex.com/discord"
pipe_func_docs = "https://docs.pipelex.com/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc"
+ backend_provider_docs = "https://docs.pipelex.com/pages/configuration/config-technical/inference-backend-config/"
diff --git a/tests/acceptance/init_command.md b/tests/acceptance/init_command.md
new file mode 100644
index 000000000..ecc7d113c
--- /dev/null
+++ b/tests/acceptance/init_command.md
@@ -0,0 +1,477 @@
+# Init Command - Acceptance Test Checklist
+
+This document provides a comprehensive checklist for manually testing all scenarios of the `pipelex init` command.
+
+## Pre-Test Setup
+
+Before each test scenario, ensure:
+
+- [ ] You have a clean test environment (consider using a temporary directory)
+- [ ] No `.pipelex/` directory exists in your test location
+- [ ] You have write permissions in the test directory
+
+## Test Scenarios
+
+### 1. First-Time Full Initialization (`pipelex init`)
+
+**Test Case 1.1: Complete initialization with default selections**
+
+- [ ] Run: `pipelex init`
+- [ ] Verify welcome panel is displayed with list of actions
+- [ ] Press Enter to confirm initialization
+- [ ] Verify config files are copied successfully
+- [ ] Verify backend selection panel is displayed with numbered options
+- [ ] Select option `1` (pipelex_inference) and press Enter
+- [ ] Verify confirmation message shows 1 backend configured
+- [ ] Verify routing profile is set to "pipelex_first"
+- [ ] Verify telemetry selection panel is displayed
+- [ ] Select option `1` (OFF) and press Enter
+- [ ] Verify telemetry mode is set successfully
+- [ ] Verify `.pipelex/` directory exists with all config files
+- [ ] Verify `.pipelex/inference/backends.toml` has pipelex_inference enabled
+- [ ] Verify `.pipelex/inference/routing_profiles.toml` has active = "pipelex_first"
+- [ ] Verify `.pipelex/telemetry.toml` has telemetry_mode = "off"
+
+**Test Case 1.2: Initialization with multiple backends**
+
+- [ ] Run: `pipelex init`
+- [ ] Confirm initialization
+- [ ] Note the displayed backend numbers, then select 3 backends (e.g., openai, anthropic, mistral) using their indices
+- [ ] Verify confirmation shows 3 backends configured
+- [ ] Verify "Setting up routing for multiple backends" message
+- [ ] Verify primary backend selection panel is displayed
+- [ ] Select primary backend (e.g., `1` for the first in your selection)
+- [ ] Verify fallback order panel is displayed
+- [ ] Press Enter to keep default order or enter custom order (e.g., `2,1`)
+- [ ] Verify "custom_routing" profile is created and set as active
+- [ ] Verify `.pipelex/inference/routing_profiles.toml` contains custom_routing profile
+- [ ] Verify fallback_order is set correctly
+
+**Test Case 1.3: Initialization with all backends**
+
+- [ ] Run: `pipelex init`
+- [ ] Confirm initialization
+- [ ] Enter `a` or `all` for backend selection
+- [ ] Verify all backends (except internal) are enabled
+- [ ] Follow through routing and telemetry setup
+- [ ] Verify all backends are enabled in backends.toml
+
+**Test Case 1.4: Cancel at backend selection**
+
+- [ ] Run: `pipelex init`
+- [ ] Confirm initialization
+- [ ] Enter `q` or `quit` at backend selection
+- [ ] Verify graceful exit with message
+- [ ] Verify config files were created but backends.toml remains in template state
+
+**Test Case 1.5: Cancel at initialization confirmation**
+
+- [ ] Run: `pipelex init`
+- [ ] Enter `n` when asked to continue
+- [ ] Verify cancellation message
+- [ ] Verify suggestion to run `pipelex init` later
+- [ ] Verify no files were created
+
+---
+
+### 2. Focused Initialization - Config Only
+
+**Test Case 2.1: Initialize config files only**
+
+- [ ] Run: `pipelex init config`
+- [ ] Confirm initialization
+- [ ] Verify config files are copied
+- [ ] Verify NO backend selection prompt appears
+- [ ] Verify NO telemetry prompt appears
+- [ ] Verify `.pipelex/` directory exists with config files
+- [ ] Verify `.pipelex/inference/backends.toml` exists in template state
+
+**Test Case 2.2: Config already initialized**
+
+- [ ] Run: `pipelex init config` (after already initialized)
+- [ ] Verify message: "Configuration files are already in place!"
+- [ ] Verify tip about using `--reset`
+- [ ] Verify no changes made
+
+---
+
+### 3. Focused Initialization - Inference Only
+
+**Test Case 3.1: Initialize inference with existing config**
+
+- [ ] Ensure `.pipelex/` exists (run `pipelex init config` first)
+- [ ] Run: `pipelex init inference`
+- [ ] Verify backend selection panel is displayed
+- [ ] Note the displayed backend numbers, then select 2 backends (e.g., openai, anthropic) using their indices
+- [ ] Verify backends are enabled in backends.toml
+- [ ] Verify routing is configured automatically
+- [ ] Verify NO telemetry prompt appears
+
+**Test Case 3.2: Initialize inference without config files**
+
+- [ ] Remove `.pipelex/` directory
+- [ ] Run: `pipelex init inference`
+- [ ] Verify warning about missing backends.toml
+- [ ] Verify command handles gracefully
+
+**Test Case 3.3: Inference already configured**
+
+- [ ] Run: `pipelex init inference` (after already configured)
+- [ ] Verify message: "Inference backends are already configured!"
+- [ ] Verify question: "Would you like to reconfigure inference backends?"
+- [ ] Enter `y` to reconfigure
+- [ ] Verify backend selection panel appears with current selection shown
+- [ ] Verify you can change backends
+- [ ] Verify routing is updated based on new selection
+
+**Test Case 3.4: Decline reconfiguration**
+
+- [ ] Run: `pipelex init inference` (after already configured)
+- [ ] Enter `n` when asked to reconfigure
+- [ ] Verify "No changes made" message
+- [ ] Verify no changes to backends.toml
+
+---
+
+### 4. Focused Initialization - Routing Only
+
+**Test Case 4.1: Configure routing with multiple backends enabled**
+
+- [ ] Ensure `.pipelex/` exists with multiple backends enabled
+- [ ] Run: `pipelex init routing`
+- [ ] Verify primary backend selection panel
+- [ ] Select primary backend
+- [ ] Verify fallback order configuration
+- [ ] Configure order and confirm
+- [ ] Verify routing_profiles.toml is updated with custom_routing
+
+**Test Case 4.2: Configure routing with single backend**
+
+- [ ] Ensure only one backend is enabled
+- [ ] Run: `pipelex init routing`
+- [ ] Verify profile is set to `all_{backend_key}`
+- [ ] Verify no primary/fallback prompts
+
+**Test Case 4.3: Configure routing with no backends enabled**
+
+- [ ] Disable all backends in backends.toml
+- [ ] Run: `pipelex init routing`
+- [ ] Verify warning: "No backends enabled. Please run 'pipelex init inference' first."
+- [ ] Verify no changes to routing_profiles.toml
+
+**Test Case 4.4: Routing already configured**
+
+- [ ] Run: `pipelex init routing` (after already configured)
+- [ ] Verify message: "Routing profile is already configured!"
+- [ ] Verify question about reconfiguration
+- [ ] Test both accepting and declining reconfiguration
+
+---
+
+### 5. Focused Initialization - Telemetry Only
+
+**Test Case 5.1: Initialize telemetry only**
+
+- [ ] Ensure `.pipelex/` exists without telemetry.toml
+- [ ] Run: `pipelex init telemetry`
+- [ ] Verify telemetry selection panel
+- [ ] Select option `2` (ANONYMOUS)
+- [ ] Verify telemetry.toml is created with mode = "anonymous"
+- [ ] Verify NO backend or routing prompts
+
+**Test Case 5.2: Telemetry already configured**
+
+- [ ] Run: `pipelex init telemetry` (after already configured)
+- [ ] Verify message: "Telemetry preferences are already configured!"
+- [ ] Verify question about reconfiguration
+- [ ] Enter `y` to reconfigure
+- [ ] Select different mode
+- [ ] Verify telemetry.toml is updated
+
+**Test Case 5.3: Cancel telemetry selection**
+
+- [ ] Run: `pipelex init telemetry`
+- [ ] Enter `q` at telemetry prompt
+- [ ] Verify graceful exit
+- [ ] Verify telemetry.toml is not created
+
+---
+
+### 6. Reset Operations
+
+**Test Case 6.1: Reset all with --reset**
+
+- [ ] Ensure `.pipelex/` fully configured
+- [ ] Run: `pipelex init --reset`
+- [ ] Verify panel shows "Resetting Configuration" (yellow)
+- [ ] Verify all items listed for reset
+- [ ] Confirm reset
+- [ ] Go through all selections again
+- [ ] Verify all files are overwritten
+- [ ] Verify new selections are applied
+
+**Test Case 6.2: Reset config only**
+
+- [ ] Run: `pipelex init config --reset`
+- [ ] Verify config files are overwritten
+- [ ] Verify backend selection prompt appears
+- [ ] Complete configuration
+- [ ] Verify backends.toml and routing are updated
+
+**Test Case 6.3: Reset inference only**
+
+- [ ] Run: `pipelex init inference --reset`
+- [ ] Verify backend selection appears
+- [ ] Change backend selection
+- [ ] Verify backends.toml is updated
+- [ ] Verify routing is reconfigured
+
+**Test Case 6.4: Reset routing only**
+
+- [ ] Run: `pipelex init routing --reset`
+- [ ] Verify routing configuration prompts
+- [ ] Change routing configuration
+- [ ] Verify routing_profiles.toml is updated
+
+**Test Case 6.5: Reset telemetry only**
+
+- [ ] Run: `pipelex init telemetry --reset`
+- [ ] Verify telemetry prompt
+- [ ] Change telemetry mode
+- [ ] Verify telemetry.toml is updated
+
+---
+
+### 7. All Already Configured
+
+**Test Case 7.1: Everything configured**
+
+- [ ] Ensure complete initialization
+- [ ] Run: `pipelex init`
+- [ ] Verify message: "Pipelex is already fully initialized!"
+- [ ] Verify tip about `--reset`
+- [ ] Verify no prompts appear
+- [ ] Verify no changes made
+
+---
+
+### 8. Input Validation
+
+**Test Case 8.1: Invalid backend selection - out of range**
+
+- [ ] Run: `pipelex init`
+- [ ] Enter invalid index like `99`
+- [ ] Verify error message with valid range
+- [ ] Verify prompt repeats
+- [ ] Enter valid selection
+- [ ] Verify successful continuation
+
+**Test Case 8.2: Invalid backend selection - non-numeric**
+
+- [ ] Run: `pipelex init`
+- [ ] Enter invalid input like `abc`
+- [ ] Verify error message
+- [ ] Verify prompt repeats
+- [ ] Enter valid selection
+
+**Test Case 8.3: Space-separated backend indices**
+
+- [ ] Run: `pipelex init`
+- [ ] Note the displayed backend numbers, then enter 3 backend indices with spaces (e.g., for openai, anthropic, mistral)
+- [ ] Verify backends are parsed correctly
+- [ ] Verify all selected backends are enabled
+
+**Test Case 8.4: Default selection (press Enter)**
+
+- [ ] Run: `pipelex init`
+- [ ] Press Enter at backend selection (accept default)
+- [ ] Verify default backend (1 - pipelex_inference) is selected
+
+**Test Case 8.5: Keep current selection**
+
+- [ ] Run: `pipelex init inference` (with existing selection)
+- [ ] Agree to reconfigure
+- [ ] Press Enter at backend selection
+- [ ] Verify current selection is kept
+
+**Test Case 8.6: Invalid telemetry selection**
+
+- [ ] Run: `pipelex init`
+- [ ] Complete to telemetry prompt
+- [ ] Enter invalid option like `5`
+- [ ] Verify error message
+- [ ] Enter valid option
+- [ ] Verify successful completion
+
+**Test Case 8.7: Invalid fallback order - wrong count**
+
+- [ ] Select 3+ backends
+- [ ] At fallback order prompt, enter only 2 indices
+- [ ] Verify error: "You must specify all X backends"
+- [ ] Enter correct count of indices
+- [ ] Verify successful ordering
+
+**Test Case 8.8: Invalid fallback order - duplicates**
+
+- [ ] Select 3+ backends
+- [ ] At fallback order prompt, enter `1 1 2`
+- [ ] Verify error: "Duplicate indices not allowed"
+- [ ] Enter valid unique indices
+- [ ] Verify successful ordering
+
+---
+
+### 9. Edge Cases
+
+**Test Case 9.1: pipelex_inference always sets pipelex_first**
+
+- [ ] Run: `pipelex init`
+- [ ] Select pipelex_inference as one of multiple backends
+- [ ] Verify routing is set to "pipelex_first" automatically
+- [ ] Verify NO primary/fallback prompts
+
+**Test Case 9.2: Single non-pipelex backend**
+
+- [ ] Run: `pipelex init`
+- [ ] Note the displayed backend numbers, then select only one backend (e.g., openai) using its index
+- [ ] Verify routing is set to "all_{backend_key}" (e.g., "all_openai")
+- [ ] Verify profile is created if it doesn't exist
+- [ ] Verify NO primary/fallback prompts
+
+**Test Case 9.3: Profile creation confirmation**
+
+- [ ] Select a single obscure backend
+- [ ] If profile doesn't exist in template, verify prompt to create
+- [ ] Accept profile creation
+- [ ] Verify profile is created in routing_profiles.toml
+
+**Test Case 9.4: Decline profile creation**
+
+- [ ] Select a backend without existing profile
+- [ ] Decline profile creation when prompted
+- [ ] Verify message: "Keeping current routing profile configuration"
+- [ ] Verify no changes to routing_profiles.toml
+
+**Test Case 9.5: Two backends (automatic fallback)**
+
+- [ ] Select exactly 2 backends
+- [ ] Select primary backend
+- [ ] Verify fallback_order is set automatically with both backends
+- [ ] Verify NO fallback ordering prompt (only 1 remaining)
+
+---
+
+### 10. Files and Directory Structure
+
+**Test Case 10.1: Verify directory structure**
+
+After complete initialization, verify:
+
+- [ ] `.pipelex/` directory exists
+- [ ] `.pipelex/pipelex.toml` exists
+- [ ] `.pipelex/inference/` directory exists
+- [ ] `.pipelex/inference/backends.toml` exists
+- [ ] `.pipelex/inference/routing_profiles.toml` exists
+- [ ] `.pipelex/telemetry.toml` exists
+- [ ] All other expected config files from template are present
+
+**Test Case 10.2: Verify file contents - backends.toml**
+
+- [ ] Open `.pipelex/inference/backends.toml`
+- [ ] Verify selected backends have `enabled = true`
+- [ ] Verify non-selected backends have `enabled = false`
+- [ ] Verify `internal` backend is always enabled
+
+**Test Case 10.3: Verify file contents - routing_profiles.toml**
+
+- [ ] Open `.pipelex/inference/routing_profiles.toml`
+- [ ] Verify `active` field matches expected profile
+- [ ] If custom_routing: verify `default` and `fallback_order` fields
+- [ ] Verify `routes` section exists
+
+**Test Case 10.4: Verify file contents - telemetry.toml**
+
+- [ ] Open `.pipelex/telemetry.toml`
+- [ ] Verify `telemetry_mode` matches selection
+- [ ] Verify valid values: "off", "anonymous", or "identified"
+
+**Test Case 10.5: Existing files not overwritten (without reset)**
+
+- [ ] Modify `.pipelex/pipelex.toml` (add a comment)
+- [ ] Run: `pipelex init config`
+- [ ] Verify your comment is preserved
+- [ ] Verify message about skipped existing files
+
+---
+
+### 11. Doctor Integration
+
+**Test Case 11.1: Doctor command suggests init**
+
+- [ ] Remove `.pipelex/` directory
+- [ ] Run: `pipelex doctor`
+- [ ] Verify doctor detects missing configuration
+- [ ] Follow suggestion to run init
+- [ ] Verify successful configuration
+
+---
+
+### 12. Performance and UX
+
+**Test Case 12.1: Responsive feedback**
+
+- [ ] Run: `pipelex init`
+- [ ] Verify all prompts appear immediately
+- [ ] Verify success messages appear after actions
+- [ ] Verify progress is clear at each step
+
+**Test Case 12.2: Clear instructions**
+
+- [ ] Run through full initialization
+- [ ] Verify all panels have clear titles
+- [ ] Verify all prompts have clear instructions
+- [ ] Verify examples are provided (e.g., "1,5,6 or 1 5 6")
+- [ ] Verify default values are clearly indicated
+
+**Test Case 12.3: Error recovery**
+
+- [ ] Make intentional errors at each prompt
+- [ ] Verify error messages are helpful
+- [ ] Verify you can recover and continue
+- [ ] Verify no need to restart from beginning
+
+---
+
+## Test Environment Cleanup
+
+After completing tests:
+
+- [ ] Remove test `.pipelex/` directories
+- [ ] Document any issues found
+- [ ] Note any unclear UX or confusing prompts
+- [ ] Record any unexpected behavior
+
+---
+
+## Sign-Off
+
+**Tester Name:** _________________________
+
+**Date:** _________________________
+
+**Test Environment:**
+
+- OS: _________________________
+- Python Version: _________________________
+- Pipelex Version: _________________________
+
+**Overall Result:** [ ] PASS [ ] FAIL
+
+**Notes:**
+
+_____________________________________________________________________________
+
+_____________________________________________________________________________
+
+_____________________________________________________________________________
+
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py b/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py
new file mode 100644
index 000000000..2cefa1f4b
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py
@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper
+
+
+class TestEdgeCases:
+ def test_pipelex_inference_sets_pipelex_first(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 9.1: pipelex_inference always sets pipelex_first."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for pipelex_inference and openai
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["pipelex_inference", "openai"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs - no primary/fallback prompts expected
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select pipelex_inference and openai
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify pipelex_first is set automatically
+ env.verify_routing("pipelex_first")
+
+ def test_single_non_pipelex_backend(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 9.2: Single non-pipelex backend."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get index for openai only
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["openai"])
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(str(indices[0])) # Select only openai
+ env.add_confirm_input(True) # Confirm creating profile if needed
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify routing is set to all_openai
+ env.verify_routing("all_openai")
+
+ def test_two_backends_automatic_fallback(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 9.5: Two backends (automatic fallback)."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for exactly 2 backends (anthropic first, then openai)
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "openai"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs - no fallback order prompt for exactly 2 backends
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select anthropic, openai
+ env.add_prompt_input("1") # Primary backend: anthropic (first in selection)
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify custom routing with automatic fallback
+ env.verify_routing("custom_routing", expected_default="anthropic", expected_fallback_order=["anthropic", "openai"])
+
+ def test_reset_all_with_flag(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 6.1: Reset all with --reset flag."""
+ # Setup environment with existing config
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # Set initial state
+ backends_path = env.inference_dir / "backends.toml"
+ toml_doc = load_toml_with_tomlkit(str(backends_path))
+ toml_doc["pipelex_inference"]["enabled"] = True # type: ignore[index]
+ save_toml_to_path(toml_doc, str(backends_path))
+
+ telemetry_path = env.pipelex_dir / "telemetry.toml"
+ toml_doc_tel = load_toml_with_tomlkit(str(telemetry_path))
+ toml_doc_tel["telemetry_mode"] = "identified"
+ save_toml_to_path(toml_doc_tel, str(telemetry_path))
+
+ # Get index for anthropic
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic"])
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm reset
+ env.add_prompt_input(str(indices[0])) # Select anthropic
+ env.add_confirm_input(True) # Confirm creating profile if needed
+ env.add_prompt_input("1") # Telemetry: OFF
+
+ env.setup_mocks()
+
+ # Execute with reset flag
+ init_cmd(focus=InitFocus.ALL, reset=True)
+
+ # Verify configuration was reset
+ env.verify_backends_enabled(["anthropic"])
+ env.verify_telemetry("off")
+
+ def test_verify_backends_toml_contents(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 10.2: Verify backends.toml contents."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for specific backends
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "mistral"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select openai, mistral
+ env.add_prompt_input("1") # Primary backend
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify detailed backends.toml contents
+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))
+
+ # Selected backends should be enabled
+ assert toml_doc["openai"]["enabled"] is True # type: ignore[index]
+ assert toml_doc["mistral"]["enabled"] is True # type: ignore[index]
+
+ # Non-selected backends should be disabled
+ assert toml_doc["pipelex_inference"]["enabled"] is False # type: ignore[index]
+ assert toml_doc["anthropic"]["enabled"] is False # type: ignore[index]
+
+ # Internal backend should be enabled
+ assert toml_doc["internal"]["enabled"] is True # type: ignore[index]
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_first_time.py b/tests/e2e/pipelex/cli/test_init_cmd_first_time.py
new file mode 100644
index 000000000..d92589109
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_init_cmd_first_time.py
@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+import typer
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit
+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper
+
+
+class TestFirstTimeInitialization:
+ def test_complete_init_with_default_selections(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 1.1: Complete initialization with default selections."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs: confirm, default backend (1), telemetry OFF (1)
+ env.add_confirm_input(True)
+ env.add_prompt_input("1") # Default: pipelex_inference
+ env.add_prompt_input("1") # Telemetry: OFF
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify
+ env.verify_file_exists("pipelex.toml")
+ env.verify_file_exists("inference/backends.toml")
+ env.verify_file_exists("inference/routing_profiles.toml")
+ env.verify_file_exists("telemetry.toml")
+ env.verify_backends_enabled(["pipelex_inference"])
+ env.verify_routing("pipelex_first")
+ env.verify_telemetry("off")
+
+ def test_init_with_multiple_backends_and_routing(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 1.2: Initialization with multiple backends."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for anthropic, mistral, openai (in this order for testing)
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "mistral", "openai"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select 3 backends
+ env.add_prompt_input("1") # Primary: first one (anthropic)
+ env.add_prompt_input("2,1") # Custom fallback order (mistral, anthropic)
+ env.add_prompt_input("2") # Telemetry: ANONYMOUS
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify backends
+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])
+
+ # Verify custom routing
+ env.verify_routing("custom_routing", expected_default="anthropic")
+
+ # Verify telemetry
+ env.verify_telemetry("anonymous")
+
+ def test_init_with_all_backends(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 1.3: Initialization with all backends."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("all") # Select all backends
+ env.add_prompt_input("1") # Telemetry: OFF
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify all backends are enabled
+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))
+ for backend_key in toml_doc:
+ if backend_key != "internal":
+ assert toml_doc[backend_key]["enabled"] is True # type: ignore[index]
+
+ # Verify routing (pipelex_first since pipelex_inference is included)
+ env.verify_routing("pipelex_first")
+
+ def test_cancel_at_backend_selection(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 1.4: Cancel at backend selection."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs: confirm, then quit at backend selection
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("q") # Quit at backend selection
+
+ env.setup_mocks()
+
+ # Execute - may raise an exit exception on cancellation
+ try:
+ init_cmd(focus=InitFocus.ALL, reset=False)
+ except (typer.Exit, SystemExit):
+ # Expected: user quit at backend selection
+ pass
+
+ # Verify config files were created but backends remain in template state
+ env.verify_file_exists("inference/backends.toml")
+
+ # Verify pipelex_inference is still enabled (default template state)
+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))
+ assert toml_doc["pipelex_inference"]["enabled"] is True # type: ignore[index]
+
+ def test_cancel_at_initialization_confirmation(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 1.5: Cancel at initialization confirmation."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs: decline confirmation
+ env.add_confirm_input(False) # Decline initialization
+
+ env.setup_mocks()
+
+ # Execute - should raise typer.Exit
+ with pytest.raises(typer.Exit):
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify no files were created
+ env.verify_file_not_exists("pipelex.toml")
+ env.verify_file_not_exists("inference/backends.toml")
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_focused.py b/tests/e2e/pipelex/cli/test_init_cmd_focused.py
new file mode 100644
index 000000000..a4e1aac3a
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_init_cmd_focused.py
@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper
+
+
+class TestFocusedInitialization:
+ def test_config_only_initialization(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 2.1: Initialize config files only."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("1") # Backend selection
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute with CONFIG focus
+ init_cmd(focus=InitFocus.CONFIG, reset=False)
+
+ # Verify config files exist
+ env.verify_file_exists("pipelex.toml")
+ env.verify_file_exists("inference/backends.toml")
+
+ def test_inference_with_existing_config(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 3.1: Initialize inference with existing config."""
+ # Setup environment with existing config
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)
+
+ # Get indices for anthropic and openai
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "openai"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs - need to confirm reconfigure since backends already exist
+ env.add_confirm_input(True) # Confirm reconfigure
+ env.add_prompt_input(indices_str) # Select 2 backends
+ env.add_prompt_input("1") # Primary backend (anthropic - first in selection)
+
+ env.setup_mocks()
+
+ # Execute with INFERENCE focus
+ init_cmd(focus=InitFocus.INFERENCE, reset=False)
+
+ # Verify backends are enabled
+ env.verify_backends_enabled(["openai", "anthropic"])
+
+ # Verify routing is configured
+ env.verify_routing("custom_routing", expected_default="anthropic")
+
+ # Verify NO telemetry prompt appeared
+ env.verify_file_not_exists("telemetry.toml")
+
+ def test_reconfigure_inference(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 3.3: Inference already configured - reconfigure."""
+ # Setup environment with existing config
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # Set pipelex_inference as initially enabled
+ backends_path = env.inference_dir / "backends.toml"
+ toml_doc = load_toml_with_tomlkit(str(backends_path))
+ toml_doc["pipelex_inference"]["enabled"] = True # type: ignore[index]
+ save_toml_to_path(toml_doc, str(backends_path))
+
+ # Get index for mistral
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["mistral"])
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm reconfigure
+ env.add_prompt_input(str(indices[0])) # Change to mistral
+ env.add_confirm_input(True) # Confirm creating profile if needed
+
+ env.setup_mocks()
+
+ # Execute with INFERENCE focus
+ init_cmd(focus=InitFocus.INFERENCE, reset=False)
+
+ # Verify backend was changed
+ env.verify_backends_enabled(["mistral"])
+
+ # Verify routing was updated
+ env.verify_routing("all_mistral")
+
+ def test_configure_routing_with_multiple_backends(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 4.1: Configure routing with multiple backends enabled."""
+ # Setup environment with existing config and multiple backends enabled
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # Enable multiple backends (anthropic, mistral, openai - in TOML order)
+ # Disable pipelex_inference so we get custom_routing instead of pipelex_first
+ backends_path = env.inference_dir / "backends.toml"
+ toml_doc = load_toml_with_tomlkit(str(backends_path))
+ toml_doc["pipelex_inference"]["enabled"] = False # type: ignore[index]
+ toml_doc["anthropic"]["enabled"] = True # type: ignore[index]
+ toml_doc["mistral"]["enabled"] = True # type: ignore[index]
+ toml_doc["openai"]["enabled"] = True # type: ignore[index]
+ save_toml_to_path(toml_doc, str(backends_path))
+
+ # User inputs for routing - need to confirm reconfigure since routing already exists
+ env.add_confirm_input(True) # Confirm reconfigure
+ # The backends are enabled and will be listed as: anthropic, mistral, openai
+ env.add_prompt_input("1") # Primary backend: first one (anthropic)
+ env.add_prompt_input("1,2") # Fallback order for remaining 2
+
+ env.setup_mocks()
+
+ # Execute with ROUTING focus
+ init_cmd(focus=InitFocus.ROUTING, reset=False)
+
+ # Verify routing was configured
+ env.verify_routing("custom_routing", expected_default="anthropic")
+
+ def test_configure_routing_with_single_backend(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 4.2: Configure routing with single backend."""
+ # Setup environment with existing config and single backend enabled
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # Enable only openai
+ backends_path = env.inference_dir / "backends.toml"
+ toml_doc = load_toml_with_tomlkit(str(backends_path))
+ toml_doc["openai"]["enabled"] = True # type: ignore[index]
+ toml_doc["pipelex_inference"]["enabled"] = False # type: ignore[index]
+ save_toml_to_path(toml_doc, str(backends_path))
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm creating profile if needed
+
+ env.setup_mocks()
+
+ # Execute with ROUTING focus
+ init_cmd(focus=InitFocus.ROUTING, reset=False)
+
+ # Verify routing is set to all_openai
+ env.verify_routing("all_openai")
+
+ def test_telemetry_only_initialization(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 5.1: Initialize telemetry only."""
+ # Setup environment with existing config but no telemetry
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("2") # Telemetry: ANONYMOUS
+
+ env.setup_mocks()
+
+ # Execute with TELEMETRY focus
+ init_cmd(focus=InitFocus.TELEMETRY, reset=False)
+
+ # Verify telemetry was created
+ env.verify_telemetry("anonymous")
+
+ def test_reconfigure_telemetry(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 5.2: Telemetry already configured - reconfigure."""
+ # Setup environment with existing telemetry
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # Set initial telemetry to OFF
+ telemetry_path = env.pipelex_dir / "telemetry.toml"
+ toml_doc = load_toml_with_tomlkit(str(telemetry_path))
+ toml_doc["telemetry_mode"] = "off"
+ save_toml_to_path(toml_doc, str(telemetry_path))
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm reconfigure
+ env.add_prompt_input("3") # Change to IDENTIFIED
+
+ env.setup_mocks()
+
+ # Execute with TELEMETRY focus
+ init_cmd(focus=InitFocus.TELEMETRY, reset=False)
+
+ # Verify telemetry was changed
+ env.verify_telemetry("identified")
+
+ def test_everything_already_configured(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 7.1: Everything configured - decline reconfigure."""
+ # Setup complete environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # User inputs: decline reconfigure - no prompts needed if everything is configured
+ # The command should detect everything is configured and exit
+
+ env.setup_mocks()
+
+ # Execute - should complete immediately since everything is configured
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Should complete without errors (no changes made)
+ # Verify files still exist
+ env.verify_file_exists("telemetry.toml")
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_validation.py b/tests/e2e/pipelex/cli/test_init_cmd_validation.py
new file mode 100644
index 000000000..1956bf77e
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_init_cmd_validation.py
@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.kit.paths import get_kit_configs_dir
+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper
+
+
+class TestInputValidation:
+ def test_invalid_backend_index_then_valid(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 8.1: Invalid backend index, then valid."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs: invalid index, then valid
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("99") # Invalid index
+ env.add_prompt_input("1") # Valid: pipelex_inference
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify successful completion with valid selection
+ env.verify_backends_enabled(["pipelex_inference"])
+
+ def test_invalid_non_numeric_input_then_valid(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 8.2: Non-numeric input, then valid."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("abc") # Invalid non-numeric
+ env.add_prompt_input("1") # Valid: pipelex_inference
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify successful completion
+ env.verify_backends_enabled(["pipelex_inference"])
+
+ def test_space_separated_backend_indices(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 8.3: Space-separated backend indices."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for openai, anthropic, mistral
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "anthropic", "mistral"])
+ indices_str = " ".join(str(i) for i in indices) # Space-separated
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Space-separated indices
+ env.add_prompt_input("1") # Primary backend
+ env.add_prompt_input("") # Accept default fallback order
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify all backends are enabled
+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])
+
+ def test_default_selection_press_enter(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 8.4: Default selection (press Enter)."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs: empty string for default
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("") # Empty = default (pipelex_inference)
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify default backend is selected
+ env.verify_backends_enabled(["pipelex_inference"])
+
+ def test_invalid_telemetry_selection_then_valid(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 8.6: Invalid telemetry selection."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("1") # Backend selection
+ env.add_prompt_input("5") # Invalid telemetry option
+ env.add_prompt_input("2") # Valid: ANONYMOUS
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify successful completion with valid telemetry
+ env.verify_telemetry("anonymous")
+
+ def test_invalid_fallback_order_wrong_count(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 8.7: Invalid fallback order - wrong count."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for 3 backends
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "anthropic", "mistral"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select 3 backends
+ env.add_prompt_input("1") # Primary backend
+ env.add_prompt_input("1") # Invalid: only 1 index instead of 2
+ env.add_prompt_input("1,2") # Valid: 2 indices for remaining backends
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify successful completion
+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])
+
+ def test_invalid_fallback_order_duplicates(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test Case 8.8: Invalid fallback order - duplicates."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for 3 backends
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "anthropic", "mistral"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select 3 backends
+ env.add_prompt_input("1") # Primary backend
+ env.add_prompt_input("1,1") # Invalid: duplicate indices
+ env.add_prompt_input("1,2") # Valid: unique indices
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify successful completion
+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])
diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py
deleted file mode 100644
index 4135a9df5..000000000
--- a/tests/helpers/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Test helper utilities package."""
diff --git a/tests/helpers/init_cmd_helpers.py b/tests/helpers/init_cmd_helpers.py
new file mode 100644
index 000000000..1eaf38e40
--- /dev/null
+++ b/tests/helpers/init_cmd_helpers.py
@@ -0,0 +1,316 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit
+
+
+def get_backend_indices_helper(backends_toml_path: str, backend_names: list[str]) -> list[int]:
+ """Get 1-based indices for given backend names from backends.toml.
+
+ Args:
+ backends_toml_path: Path to backends.toml file.
+ backend_names: List of backend keys to find indices for.
+
+ Returns:
+ List of 1-based indices corresponding to the backend names.
+ """
+ toml_doc = load_toml_with_tomlkit(backends_toml_path)
+ backend_list: list[str] = [key for key in toml_doc if key != "internal"]
+ indices: list[int] = []
+ for name in backend_names:
+ if name in backend_list:
+ # 1-based index for user input
+ indices.append(backend_list.index(name) + 1)
+ return indices
+
+
+def verify_backends_toml(backends_toml_path: str, expected_enabled: list[str]) -> None:
+ """Verify backends.toml has expected backends enabled.
+
+ Args:
+ backends_toml_path: Path to backends.toml file.
+ expected_enabled: List of backend keys expected to be enabled.
+ """
+ toml_doc = load_toml_with_tomlkit(backends_toml_path)
+
+ for backend_key in toml_doc:
+ if backend_key == "internal":
+ # Internal should always be enabled
+ assert toml_doc[backend_key]["enabled"] is True # type: ignore[index]
+ elif backend_key in expected_enabled:
+ assert toml_doc[backend_key]["enabled"] is True, f"Expected {backend_key} to be enabled" # type: ignore[index]
+ # Check if enabled field exists before asserting
+ elif "enabled" in toml_doc[backend_key]: # type: ignore[operator]
+ assert toml_doc[backend_key]["enabled"] is False, f"Expected {backend_key} to be disabled" # type: ignore[index]
+
+
+def verify_routing_profile(
+ routing_profiles_toml_path: str,
+ expected_active: str,
+ expected_default: str | None = None,
+ expected_fallback_order: list[str] | None = None,
+) -> None:
+ """Verify routing_profiles.toml has expected configuration.
+
+ Args:
+ routing_profiles_toml_path: Path to routing_profiles.toml file.
+ expected_active: Expected active profile name.
+ expected_default: Expected default backend (optional).
+ expected_fallback_order: Expected fallback order (optional).
+ """
+ toml_doc = load_toml_with_tomlkit(routing_profiles_toml_path)
+
+ assert toml_doc["active"] == expected_active, f"Expected active profile '{expected_active}', got '{toml_doc['active']}'"
+
+ if expected_default is not None or expected_fallback_order is not None:
+ profiles: dict[str, Any] = toml_doc.get("profiles", {}) # type: ignore[assignment]
+ assert expected_active in profiles, f"Profile '{expected_active}' not found"
+
+ profile = profiles[expected_active] # type: ignore[index]
+
+ if expected_default is not None:
+ assert profile.get("default") == expected_default, f"Expected default '{expected_default}'" # type: ignore[union-attr]
+
+ if expected_fallback_order is not None:
+ actual_fallback = profile.get("fallback_order") # type: ignore[union-attr]
+ assert actual_fallback == expected_fallback_order, f"Expected fallback_order {expected_fallback_order}, got {actual_fallback}"
+
+
+def verify_telemetry_config(telemetry_config_path: str, expected_mode: str) -> None:
+ """Verify telemetry.toml has expected mode.
+
+ Args:
+ telemetry_config_path: Path to telemetry.toml file.
+ expected_mode: Expected telemetry mode ("off", "anonymous", or "identified").
+ """
+ toml_doc = load_toml_with_tomlkit(telemetry_config_path)
+ assert toml_doc["telemetry_mode"] == expected_mode, f"Expected telemetry_mode '{expected_mode}', got '{toml_doc['telemetry_mode']}'"
+
+
+def setup_pipelex_dir(tmp_path: Path, copy_configs: bool = True, copy_backends: bool = True, copy_routing: bool = True) -> Path:
+ """Set up a test .pipelex directory structure.
+
+ Args:
+ tmp_path: Pytest temporary path fixture.
+ copy_configs: Whether to copy config files from kit.
+ copy_backends: Whether to copy backends.toml.
+ copy_routing: Whether to copy routing_profiles.toml.
+
+ Returns:
+ Path to created .pipelex directory.
+ """
+ pipelex_dir = tmp_path / ".pipelex"
+ pipelex_dir.mkdir(exist_ok=True)
+
+ inference_dir = pipelex_dir / "inference"
+ inference_dir.mkdir(exist_ok=True)
+
+ kit_configs_dir = Path(str(get_kit_configs_dir()))
+
+ if copy_configs:
+ # Copy main pipelex.toml
+ if (kit_configs_dir / "pipelex.toml").exists():
+ shutil.copy2(kit_configs_dir / "pipelex.toml", pipelex_dir / "pipelex.toml")
+
+ if copy_backends:
+ # Copy backends.toml
+ backends_source = kit_configs_dir / "inference" / "backends.toml"
+ if backends_source.exists():
+ shutil.copy2(backends_source, inference_dir / "backends.toml")
+
+ if copy_routing:
+ # Copy routing_profiles.toml
+ routing_source = kit_configs_dir / "inference" / "routing_profiles.toml"
+ if routing_source.exists():
+ shutil.copy2(routing_source, inference_dir / "routing_profiles.toml")
+
+ return pipelex_dir
+
+
+class MockedInitEnvironment:
+ """Encapsulates all mocking setup for init command testing.
+
+ This class manages:
+ - File system setup (tmp_path, .pipelex directory)
+ - Mocking of Console, Prompt, Confirm
+ - User input simulation
+ - Configuration verification
+ """
+
+ def __init__(self, tmp_path: Path, mocker: MockerFixture):
+ """Initialize mocked environment.
+
+ Args:
+ tmp_path: Pytest temporary path fixture.
+ mocker: Pytest mocker fixture.
+ """
+ self.tmp_path = tmp_path
+ self.mocker = mocker
+ self.pipelex_dir = tmp_path / ".pipelex"
+ self.inference_dir = self.pipelex_dir / "inference"
+
+ # Store mocks for later use
+ self.mock_console: Any = None
+ self.mock_prompt_ask: Any = None
+ self.mock_confirm_ask: Any = None
+ self.mock_config_manager: Any = None
+
+ # Input sequences
+ self.prompt_inputs: list[str] = []
+ self.confirm_inputs: list[bool] = []
+
+ def setup_empty_dir(self) -> None:
+ """Set up an empty .pipelex directory structure."""
+ self.pipelex_dir.mkdir(exist_ok=True)
+ self.inference_dir.mkdir(exist_ok=True)
+
+ def setup_with_configs(self, include_backends: bool = True, include_routing: bool = True, include_telemetry: bool = False) -> None:
+ """Set up .pipelex directory with config files from kit.
+
+ Args:
+ include_backends: Whether to include backends.toml.
+ include_routing: Whether to include routing_profiles.toml.
+ include_telemetry: Whether to include telemetry.toml.
+ """
+ setup_pipelex_dir(
+ self.tmp_path,
+ copy_configs=True,
+ copy_backends=include_backends,
+ copy_routing=include_routing,
+ )
+
+ if include_telemetry:
+ kit_configs_dir = Path(str(get_kit_configs_dir()))
+ telemetry_source = kit_configs_dir / "telemetry.toml"
+ if telemetry_source.exists():
+ shutil.copy2(telemetry_source, self.pipelex_dir / "telemetry.toml")
+
+ def mock_config_manager_paths(self) -> None:
+ """Mock config_manager to use tmp_path."""
+ self.mock_config_manager = self.mocker.MagicMock()
+ self.mock_config_manager.pipelex_config_dir = str(self.pipelex_dir)
+
+ # Patch all locations where config_manager is used
+ self.mocker.patch("pipelex.cli.commands.init.command.config_manager", self.mock_config_manager)
+ self.mocker.patch("pipelex.cli.commands.init.backends.config_manager", self.mock_config_manager)
+ self.mocker.patch("pipelex.cli.commands.init.routing.config_manager", self.mock_config_manager)
+ self.mocker.patch("pipelex.cli.commands.init.config_files.config_manager", self.mock_config_manager)
+
+ def mock_console_outputs(self) -> None:
+ """Mock Console to suppress output."""
+ self.mock_console = self.mocker.patch("pipelex.cli.commands.init.command.Console")
+ self.mocker.patch("pipelex.cli.commands.init.backends.Console")
+ self.mocker.patch("pipelex.cli.commands.init.routing.Console")
+ self.mocker.patch("pipelex.cli.commands.init.telemetry.Console")
+ self.mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Console")
+ self.mocker.patch("pipelex.cli.commands.init.ui.routing_ui.Console")
+ self.mocker.patch("pipelex.cli.commands.init.ui.telemetry_ui.Console")
+
+ def add_prompt_input(self, value: str) -> None:
+ """Add a prompt input to the sequence.
+
+ Args:
+ value: The input value to simulate.
+ """
+ self.prompt_inputs.append(value)
+
+ def add_confirm_input(self, value: bool) -> None:
+ """Add a confirm input to the sequence.
+
+ Args:
+ value: The boolean confirmation to simulate.
+ """
+ self.confirm_inputs.append(value)
+
+ def setup_mocks(self) -> None:
+ """Set up all mocks with configured inputs."""
+ self.mock_config_manager_paths()
+ self.mock_console_outputs()
+
+ # Mock Prompt.ask with side_effect for sequential inputs
+ if self.prompt_inputs:
+ self.mock_prompt_ask = self.mocker.patch(
+ "rich.prompt.Prompt.ask",
+ side_effect=self.prompt_inputs,
+ )
+
+ # Mock Confirm.ask with side_effect for sequential inputs
+ if self.confirm_inputs:
+ self.mock_confirm_ask = self.mocker.patch(
+ "rich.prompt.Confirm.ask",
+ side_effect=self.confirm_inputs,
+ )
+
+ # Mock typer.echo to suppress output
+ self.mocker.patch("typer.echo")
+
+ def get_backend_indices(self, backend_names: list[str]) -> list[int]:
+ """Get 1-based indices for backend names.
+
+ Args:
+ backend_names: List of backend keys.
+
+ Returns:
+ List of 1-based indices.
+ """
+ backends_path = str(self.inference_dir / "backends.toml")
+ return get_backend_indices_helper(backends_path, backend_names)
+
+ def verify_backends_enabled(self, expected_enabled: list[str]) -> None:
+ """Verify backends.toml has expected backends enabled.
+
+ Args:
+ expected_enabled: List of backend keys expected to be enabled.
+ """
+ backends_path = str(self.inference_dir / "backends.toml")
+ verify_backends_toml(backends_path, expected_enabled)
+
+ def verify_routing(
+ self,
+ expected_active: str,
+ expected_default: str | None = None,
+ expected_fallback_order: list[str] | None = None,
+ ) -> None:
+ """Verify routing profile configuration.
+
+ Args:
+ expected_active: Expected active profile name.
+ expected_default: Expected default backend (optional).
+ expected_fallback_order: Expected fallback order (optional).
+ """
+ routing_path = str(self.inference_dir / "routing_profiles.toml")
+ verify_routing_profile(routing_path, expected_active, expected_default, expected_fallback_order)
+
+ def verify_telemetry(self, expected_mode: str) -> None:
+ """Verify telemetry configuration.
+
+ Args:
+ expected_mode: Expected telemetry mode.
+ """
+ telemetry_path = str(self.pipelex_dir / "telemetry.toml")
+ verify_telemetry_config(telemetry_path, expected_mode)
+
+ def verify_file_exists(self, relative_path: str) -> None:
+ """Verify a file exists in the pipelex directory.
+
+ Args:
+ relative_path: Path relative to .pipelex directory.
+ """
+ file_path = self.pipelex_dir / relative_path
+ assert file_path.exists(), f"Expected file {file_path} to exist"
+
+ def verify_file_not_exists(self, relative_path: str) -> None:
+ """Verify a file does not exist in the pipelex directory.
+
+ Args:
+ relative_path: Path relative to .pipelex directory.
+ """
+ file_path = self.pipelex_dir / relative_path
+ assert not file_path.exists(), f"Expected file {file_path} to not exist"
diff --git a/tests/integration/pipelex/cli/test_init_cmd.py b/tests/integration/pipelex/cli/test_init_cmd_backend_customization.py
similarity index 68%
rename from tests/integration/pipelex/cli/test_init_cmd.py
rename to tests/integration/pipelex/cli/test_init_cmd_backend_customization.py
index 67433563c..089390851 100644
--- a/tests/integration/pipelex/cli/test_init_cmd.py
+++ b/tests/integration/pipelex/cli/test_init_cmd_backend_customization.py
@@ -1,5 +1,3 @@
-"""Integration tests for init command backend customization."""
-
from __future__ import annotations
import shutil
@@ -9,14 +7,33 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from pytest_mock import MockerFixture
-from pipelex.cli.commands.init_cmd import customize_backends_config, init_config
-from pipelex.kit.paths import get_configs_dir
+from pipelex.cli.commands.init.backends import customize_backends_config
+from pipelex.cli.commands.init.config_files import init_config
+from pipelex.kit.paths import get_kit_configs_dir
from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit
-class TestBackendCustomization:
- """Integration tests for backend customization during initialization."""
+def get_backend_indices(backends_toml_path: str, backend_names: list[str]) -> list[int]:
+ """Get 1-based indices for given backend names from backends.toml.
+
+ Args:
+ backends_toml_path: Path to backends.toml file.
+ backend_names: List of backend keys to find indices for.
+
+ Returns:
+ List of 1-based indices corresponding to the backend names.
+ """
+ toml_doc = load_toml_with_tomlkit(backends_toml_path)
+ backend_list: list[str] = [key for key in toml_doc if key != "internal"]
+ indices: list[int] = []
+ for name in backend_names:
+ if name in backend_list:
+ # 1-based index for user input
+ indices.append(backend_list.index(name) + 1)
+ return indices
+
+class TestBackendCustomization:
def test_customize_backends_config_with_default_selection(self, tmp_path: Path, mocker: MockerFixture) -> None:
"""Test backend customization with default selection (pipelex_inference)."""
# Setup directories with actual backends.toml
@@ -24,18 +41,18 @@ class TestBackendCustomization:
inference_dir.mkdir(parents=True)
# Copy actual backends.toml from kit
- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
test_backends = inference_dir / "backends.toml"
shutil.copy2(actual_backends, test_backends)
# Mock config_manager and user input
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)
# Mock Console and Prompt to simulate user selecting default [1]
- mocker.patch("pipelex.cli.commands.init_cmd.Console")
- mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="1")
+ mocker.patch("pipelex.cli.commands.init.backends.Console")
+ mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Prompt.ask", return_value="1")
# Execute
customize_backends_config()
@@ -61,18 +78,23 @@ class TestBackendCustomization:
inference_dir = tmp_path / ".pipelex" / "inference"
inference_dir.mkdir(parents=True)
- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
test_backends = inference_dir / "backends.toml"
shutil.copy2(actual_backends, test_backends)
+ # Dynamically get indices for the backends we want to test
+ backend_names = ["openai", "anthropic", "mistral"]
+ indices = get_backend_indices(str(actual_backends), backend_names)
+ indices_str = ",".join(str(i) for i in indices)
+
# Mock config_manager
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)
- # Mock user input: select [6,7,8] (openai, anthropic, mistral) in 1-based indexing
- mocker.patch("pipelex.cli.commands.init_cmd.Console")
- mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="6,7,8")
+ # Mock user input with dynamically determined indices
+ mocker.patch("pipelex.cli.commands.init.backends.Console")
+ mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Prompt.ask", return_value=indices_str)
# Execute
customize_backends_config()
@@ -99,17 +121,22 @@ class TestBackendCustomization:
inference_dir = tmp_path / ".pipelex" / "inference"
inference_dir.mkdir(parents=True)
- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
test_backends = inference_dir / "backends.toml"
shutil.copy2(actual_backends, test_backends)
+ # Dynamically get indices for the backends we want to test
+ backend_names = ["pipelex_inference", "openai", "fal"]
+ indices = get_backend_indices(str(actual_backends), backend_names)
+ indices_str = " ".join(str(i) for i in indices)
+
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)
- # Mock user input with spaces: "1 6 12" in 1-based indexing
- mocker.patch("pipelex.cli.commands.init_cmd.Console")
- mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="1 6 12")
+ # Mock user input with spaces and dynamically determined indices
+ mocker.patch("pipelex.cli.commands.init.backends.Console")
+ mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Prompt.ask", return_value=indices_str)
# Execute
customize_backends_config()
@@ -136,7 +163,7 @@ class TestBackendCustomization:
inference_dir = kit_configs_dir / "inference"
inference_dir.mkdir()
- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"
+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
shutil.copy2(actual_backends, inference_dir / "backends.toml")
# Setup target directory
@@ -144,10 +171,10 @@ class TestBackendCustomization:
target_dir.mkdir()
# Mock config_manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=str(kit_configs_dir))
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=str(kit_configs_dir))
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
mocker.patch("typer.echo")
# Execute init_config
@@ -176,9 +203,9 @@ class TestBackendCustomization:
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(config_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)
- mock_console = mocker.patch("pipelex.cli.commands.init_cmd.Console")
+ mock_console = mocker.patch("pipelex.cli.commands.init.backends.Console")
# Execute - should not raise exception
customize_backends_config()
diff --git a/tests/integration/pipelex/cli/test_init_cmd_integration.py b/tests/integration/pipelex/cli/test_init_cmd_integration.py
new file mode 100644
index 000000000..59a8ed7e2
--- /dev/null
+++ b/tests/integration/pipelex/cli/test_init_cmd_integration.py
@@ -0,0 +1,301 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper
+
+
+class TestInitCommandIntegration:
+ def test_init_with_all_backends_selection(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test initialization with 'all' backends selection."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs: confirm init, select all backends, select telemetry OFF
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("all") # Select all backends
+ env.add_prompt_input("1") # Telemetry: OFF
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify all backends are enabled (except internal which is separate)
+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))
+ for backend_key in toml_doc:
+ if backend_key != "internal":
+ assert toml_doc[backend_key]["enabled"] is True # type: ignore[index]
+
+ # Verify routing was set to pipelex_first (since pipelex_inference is in "all")
+ env.verify_routing("pipelex_first")
+
+ # Verify telemetry
+ env.verify_telemetry("off")
+
+ def test_init_with_multiple_backends_custom_routing(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test initialization with multiple backends and custom routing."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for anthropic, mistral, openai (they will be selected in this order)
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "mistral", "openai"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select anthropic, mistral, openai
+ env.add_prompt_input("1") # Primary backend: first one (anthropic)
+ env.add_prompt_input("") # Accept default fallback order
+ env.add_prompt_input("2") # Telemetry: ANONYMOUS
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify backends
+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])
+
+ # Verify custom routing was created
+ env.verify_routing("custom_routing", expected_default="anthropic")
+
+ # Verify telemetry
+ env.verify_telemetry("anonymous")
+
+ def test_init_with_single_backend_auto_routing(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test initialization with single backend sets auto routing."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get index for openai
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["openai"])
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(str(indices[0])) # Select only openai
+ env.add_confirm_input(True) # Confirm creating profile if needed
+ env.add_prompt_input("3") # Telemetry: IDENTIFIED
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify only openai is enabled
+ env.verify_backends_enabled(["openai"])
+
+ # Verify routing is set to all_openai
+ env.verify_routing("all_openai")
+
+ # Verify telemetry
+ env.verify_telemetry("identified")
+
+ def test_init_config_only_focus(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test focused initialization: config files only."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # User inputs - CONFIG focus still runs full init on first time
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("1") # Backend selection (pipelex_inference)
+ env.add_prompt_input("1") # Telemetry
+
+ env.setup_mocks()
+
+ # Execute with CONFIG focus
+ init_cmd(focus=InitFocus.CONFIG, reset=False)
+
+ # Verify config files exist
+ env.verify_file_exists("pipelex.toml")
+ env.verify_file_exists("inference/backends.toml")
+
+ # CONFIG focus on first init triggers full flow
+ env.verify_backends_enabled(["pipelex_inference"])
+ env.verify_routing("pipelex_first")
+ # Telemetry should be created
+ if (env.pipelex_dir / "telemetry.toml").exists():
+ env.verify_telemetry("off")
+
+ def test_init_inference_only_focus(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test focused initialization: inference only."""
+ # Setup environment with existing config
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)
+
+ # Get indices for anthropic and mistral
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "mistral"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs - need to confirm reconfigure since backends already exist
+ env.add_confirm_input(True) # Confirm reconfigure
+ env.add_prompt_input(indices_str) # Select anthropic, mistral
+ env.add_prompt_input("1") # Primary backend: first one (anthropic)
+ env.add_prompt_input("") # Accept default fallback order
+
+ env.setup_mocks()
+
+ # Execute with INFERENCE focus
+ init_cmd(focus=InitFocus.INFERENCE, reset=False)
+
+ # Verify backends
+ env.verify_backends_enabled(["anthropic", "mistral"])
+
+ # Verify routing was configured (anthropic is first in selection)
+ env.verify_routing("custom_routing", expected_default="anthropic")
+
+ # Verify telemetry was NOT created (inference focus only)
+ env.verify_file_not_exists("telemetry.toml")
+
+ def test_init_telemetry_only_focus(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test focused initialization: telemetry only."""
+ # Setup environment with existing config but no telemetry
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("2") # Telemetry: ANONYMOUS
+
+ env.setup_mocks()
+
+ # Execute with TELEMETRY focus
+ init_cmd(focus=InitFocus.TELEMETRY, reset=False)
+
+ # Verify telemetry was created
+ env.verify_telemetry("anonymous")
+
+ def test_init_with_reset_flag(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test initialization with reset flag overwrites existing config."""
+ # Setup environment with existing config
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # Modify telemetry to a different value
+ telemetry_path = env.pipelex_dir / "telemetry.toml"
+ toml_doc = load_toml_with_tomlkit(str(telemetry_path))
+ toml_doc["telemetry_mode"] = "identified"
+ save_toml_to_path(toml_doc, str(telemetry_path))
+
+ # Get index for mistral
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["mistral"])
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm reset
+ env.add_prompt_input(str(indices[0])) # Select mistral only
+ env.add_confirm_input(True) # Confirm creating profile if needed
+ env.add_prompt_input("1") # Telemetry: OFF
+
+ env.setup_mocks()
+
+ # Execute with reset flag
+ init_cmd(focus=InitFocus.ALL, reset=True)
+
+ # Verify backends were reset
+ env.verify_backends_enabled(["mistral"])
+
+ # Verify telemetry was reset to OFF
+ env.verify_telemetry("off")
+
+ def test_init_reconfigure_existing_inference(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test reconfiguring inference when already configured."""
+ # Setup environment with existing config
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+
+ # Set pipelex_inference as enabled initially
+ backends_path = env.inference_dir / "backends.toml"
+ toml_doc = load_toml_with_tomlkit(str(backends_path))
+ toml_doc["pipelex_inference"]["enabled"] = True # type: ignore[index]
+ save_toml_to_path(toml_doc, str(backends_path))
+
+ # Get index for openai
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["openai"])
+
+ # User inputs
+ env.add_confirm_input(True) # Confirm reconfigure
+ env.add_prompt_input(str(indices[0])) # Change to openai
+ env.add_confirm_input(True) # Confirm creating profile if needed
+
+ env.setup_mocks()
+
+ # Execute with INFERENCE focus
+ init_cmd(focus=InitFocus.INFERENCE, reset=False)
+
+ # Verify backend was changed
+ env.verify_backends_enabled(["openai"])
+
+ # Verify routing was updated
+ env.verify_routing("all_openai")
+
+ def test_init_two_backends_automatic_fallback(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test that two backends automatically set fallback without prompting order."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for exactly 2 backends (anthropic first, then openai)
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "openai"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs - note: no fallback order prompt for 2 backends
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select anthropic, openai
+ env.add_prompt_input("1") # Primary backend: anthropic (first in selection)
+ env.add_prompt_input("1") # Telemetry: OFF
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify backends
+ env.verify_backends_enabled(["openai", "anthropic"])
+
+ # Verify custom routing with automatic fallback order
+ env.verify_routing("custom_routing", expected_default="anthropic", expected_fallback_order=["anthropic", "openai"])
+
+ def test_init_pipelex_inference_sets_pipelex_first(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """Test that selecting pipelex_inference automatically sets pipelex_first routing."""
+ # Setup environment
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+
+ # Get indices for pipelex_inference and openai
+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"
+ indices = get_backend_indices_helper(str(kit_backends), ["pipelex_inference", "openai"])
+ indices_str = ",".join(str(i) for i in indices)
+
+ # User inputs - no primary/fallback prompts because pipelex_inference is included
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input(indices_str) # Select pipelex_inference and openai
+ env.add_prompt_input("1") # Telemetry: OFF
+
+ env.setup_mocks()
+
+ # Execute
+ init_cmd(focus=InitFocus.ALL, reset=False)
+
+ # Verify backends
+ env.verify_backends_enabled(["pipelex_inference", "openai"])
+
+ # Verify routing is automatically set to pipelex_first
+ env.verify_routing("pipelex_first")
diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.py
index 6653099a2..d82896fb8 100644
--- a/tests/integration/pipelex/conftest.py
+++ b/tests/integration/pipelex/conftest.py
@@ -49,7 +49,8 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
# "gpt-5-chat",
# "claude-4.5-sonnet",
# "claude-4.1-opus",
- "claude-4.5-sonnet",
+ "claude-4.5-haiku",
+ # "claude-4.5-sonnet",
# "grok-3",
# "grok-3-mini",
# "base-claude",
diff --git a/tests/integration/pipelex/plugins/conftest.py b/tests/integration/pipelex/plugins/conftest.py
index 19a5312f2..9f8a8a13a 100644
--- a/tests/integration/pipelex/plugins/conftest.py
+++ b/tests/integration/pipelex/plugins/conftest.py
@@ -1,5 +1,17 @@
import pytest
+from pipelex.cogt.exceptions import ModelManagerError
+from pipelex.hub import get_models_manager
+
+
+def is_backend_available(backend_name: str) -> bool:
+ """Check if an inference backend is available/enabled."""
+ try:
+ get_models_manager().get_required_inference_backend(backend_name)
+ return True
+ except ModelManagerError:
+ return False
+
@pytest.fixture(params=["anthropic", "mistral", "meta", "amazon"])
def bedrock_provider(request: pytest.FixtureRequest) -> str:
diff --git a/tests/integration/pipelex/plugins/test_anthropic.py b/tests/integration/pipelex/plugins/test_anthropic.py
index dc53af96a..0c03c4256 100644
--- a/tests/integration/pipelex/plugins/test_anthropic.py
+++ b/tests/integration/pipelex/plugins/test_anthropic.py
@@ -9,6 +9,7 @@ from pipelex.plugins.anthropic.anthropic_exceptions import AnthropicSDKUnsupport
from pipelex.plugins.anthropic.anthropic_llms import anthropic_list_available_models
from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.system.environment import all_env_vars_are_set, any_env_var_is_placeholder
+from tests.integration.pipelex.plugins.conftest import is_backend_available
REQUIRED_ENV_VARS = ["ANTHROPIC_API_KEY"]
@@ -26,12 +27,14 @@ class TestAnthropic:
pytestconfig: pytest.Config,
plugin_for_anthropic: Plugin,
):
+ if not is_backend_available(plugin_for_anthropic.backend):
+ pytest.skip(f"Backend '{plugin_for_anthropic.backend}' is not available or enabled")
if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
try:
- backend = get_models_manager().get_required_inference_backend("anthropic")
+ backend = get_models_manager().get_required_inference_backend(plugin_for_anthropic.backend)
anthropic_models_list = await anthropic_list_available_models(
plugin=plugin_for_anthropic,
backend=backend,
diff --git a/tests/integration/pipelex/plugins/test_mistral.py b/tests/integration/pipelex/plugins/test_mistral.py
index 23dca8dab..840a1a94b 100644
--- a/tests/integration/pipelex/plugins/test_mistral.py
+++ b/tests/integration/pipelex/plugins/test_mistral.py
@@ -6,6 +6,7 @@ from rich.table import Table
from pipelex import pretty_print
from pipelex.plugins.mistral.mistral_llms import mistral_list_available_models
from pipelex.system.environment import all_env_vars_are_set, any_env_var_is_placeholder
+from tests.integration.pipelex.plugins.conftest import is_backend_available
REQUIRED_ENV_VARS = ["MISTRAL_API_KEY"]
@@ -17,6 +18,8 @@ class TestMistral:
# pytest -k test_mistral_list_models -s -vv
# make t VERBOSE=2 TEST=test_mistral_list_models
def test_mistral_list_models(self, pytestconfig: pytest.Config):
+ if not is_backend_available("mistral"):
+ pytest.skip("Backend 'mistral' is not available or enabled")
if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
@@ -49,6 +52,8 @@ class TestMistral:
# pytest -k test_mistral_list_model_ids -s -vv
# make t VERBOSE=2 TEST=test_mistral_list_model_ids
def test_mistral_list_model_ids(self, pytestconfig: pytest.Config):
+ if not is_backend_available("mistral"):
+ pytest.skip("Backend 'mistral' is not available or enabled")
if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
diff --git a/tests/integration/pipelex/plugins/test_openai.py b/tests/integration/pipelex/plugins/test_openai.py
index fdd012a34..8e91ee774 100644
--- a/tests/integration/pipelex/plugins/test_openai.py
+++ b/tests/integration/pipelex/plugins/test_openai.py
@@ -5,6 +5,7 @@ from pipelex.hub import get_models_manager
from pipelex.plugins.openai.openai_llms import openai_list_available_models
from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.system.environment import all_env_vars_are_set, any_env_var_is_placeholder
+from tests.integration.pipelex.plugins.conftest import is_backend_available
# make t VERBOSE=2 TEST=TestOpenAI
@@ -18,6 +19,8 @@ class TestOpenAI:
pytestconfig: pytest.Config,
plugin_for_openai: Plugin,
):
+ if not is_backend_available(plugin_for_openai.backend):
+ pytest.skip(f"Backend '{plugin_for_openai.backend}' is not available or enabled")
match plugin_for_openai.backend:
case "openai":
required_env_vars = ["OPENAI_API_KEY"]
diff --git a/tests/integration/pipelex/plugins/test_openai_image_gen.py b/tests/integration/pipelex/plugins/test_openai_image_gen.py
index 7043b3c86..539c9a5d1 100644
--- a/tests/integration/pipelex/plugins/test_openai_image_gen.py
+++ b/tests/integration/pipelex/plugins/test_openai_image_gen.py
@@ -7,6 +7,7 @@ from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.tools.misc.base_64_utils import save_base_64_str_to_binary_file
from pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_path
from tests.conftest import TEST_OUTPUTS_DIR
+from tests.integration.pipelex.plugins.conftest import is_backend_available
from tests.integration.pipelex.test_data import ImageGenTestCases
@@ -16,6 +17,8 @@ from tests.integration.pipelex.test_data import ImageGenTestCases
class TestImgGenByOpenAIGpt:
@pytest.mark.parametrize(("topic", "image_desc"), ImageGenTestCases.IMAGE_DESC)
async def test_gpt_image_generation(self, topic: str, image_desc: str):
+ if not is_backend_available("openai"):
+ pytest.skip("Backend 'openai' is not available or enabled")
backend = get_models_manager().get_required_inference_backend("openai")
client = OpenAIFactory.make_openai_client(
Plugin(sdk="openai", backend="openai"),
diff --git a/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx b/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx
index a3e3645a8..0579c20e0 100644
--- a/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx
+++ b/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx
@@ -25,7 +25,7 @@ type = "PipeLLM"
description = "Describe what is in the image"
inputs = { imageA = "Image", imageB = "Image" }
output = "BasicDescription"
-model = "llm_to_extract_diagram"
+model = "llm_for_diagram_to_text"
prompt = """
Describe what you see in $imageA only.
Completely ignore $imageB.
@@ -36,7 +36,7 @@ type = "PipeLLM"
description = "Describe what is in the image"
inputs = { imageA = "Image", imageB = "Image" }
output = "BasicDescription"
-model = "llm_to_extract_diagram"
+model = "llm_for_diagram_to_text"
prompt = """
Describe what you see in $imageB only.
Completely ignore $imageA.
@@ -48,7 +48,7 @@ type = "PipeLLM"
description = "Provide detailed analysis of the image"
inputs = { image = "Photo" }
output = "VisionAnalysis"
-model = "llm_to_extract_diagram"
+model = "llm_for_diagram_to_text"
system_prompt = "You are an expert image analyst. Provide detailed, accurate descriptions."
prompt = """
Analyze this image and describe what's the main topic etc.
diff --git a/tests/test_pipelines/tricky_questions.plx b/tests/test_pipelines/tricky_questions.plx
index 70a0c931c..96a117249 100644
--- a/tests/test_pipelines/tricky_questions.plx
+++ b/tests/test_pipelines/tricky_questions.plx
@@ -16,7 +16,7 @@ type = "PipeLLM"
description = "Analyze a question to determine whether it's straightforward or tricky"
inputs = { question = "Question" }
output = "QuestionAnalysis"
-model = "llm_for_complex_reasoning"
+model = "llm_to_engineer"
prompt = """
Here is a question for an LLM:
@question
@@ -32,7 +32,7 @@ type = "PipeLLM"
description = "Answer knowingly after analyzing a question"
inputs = { question = "Question", question_analysis = "QuestionAnalysis" }
output = "ThoughtfulAnswer"
-model = "llm_for_complex_reasoning"
+model = "llm_to_engineer"
prompt = """
A question was asked:
@question
diff --git a/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.py b/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.py
index aa5f52ac9..e255b7ac2 100644
--- a/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.py
+++ b/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.py
@@ -54,7 +54,7 @@ class PipeLLMTestCases:
inputs={"image": "Image"},
output="Text",
prompt="Analyze the image: $image",
- llm_skill=LLMSkill.LLM_FOR_BASIC_VISION,
+ llm_skill=LLMSkill.LLM_FOR_IMG_TO_TEXT_CHEAP,
),
PipeLLMBlueprint(
source=None,
@@ -63,7 +63,7 @@ class PipeLLMTestCases:
inputs={"image": "Image"},
output="Text",
prompt="Analyze the image: $image",
- model=LLMSkill.LLM_FOR_BASIC_VISION,
+ model=LLMSkill.LLM_FOR_IMG_TO_TEXT_CHEAP,
),
)
@@ -75,13 +75,13 @@ class PipeLLMTestCases:
inputs={},
output="Text",
prompt="Generate text",
- llm_skill=LLMSkill.LLM_TO_WRITE_QUESTIONS,
+ llm_skill=LLMSkill.LLM_FOR_WRITING_CHEAP,
),
PipeLLMBlueprint(
source=None,
type="PipeLLM",
description="Generate with preset",
- model=LLMSkill.LLM_TO_WRITE_QUESTIONS,
+ model=LLMSkill.LLM_FOR_WRITING_CHEAP,
output="Text",
prompt="Generate text",
),
@@ -95,13 +95,13 @@ class PipeLLMTestCases:
inputs={},
output="Text",
prompt="Generate text",
- llm_skill=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,
+ llm_skill=LLMSkill.LLM_TO_ANSWER_QUESTIONS_CHEAP,
),
PipeLLMBlueprint(
source=None,
type="PipeLLM",
description="Generate with settings",
- model=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,
+ model=LLMSkill.LLM_TO_ANSWER_QUESTIONS_CHEAP,
output="Text",
prompt="Generate text",
),
@@ -116,7 +116,7 @@ class PipeLLMTestCases:
output="Analysis",
system_prompt="You are a data analyst",
prompt="Analyze: @data",
- llm_skill=LLMSkill.LLM_TO_ANALYZE_DATA,
+ llm_skill=LLMSkill.LLM_TO_ANSWER_QUESTIONS,
),
PipeLLMBlueprint(
source=None,
@@ -126,7 +126,7 @@ class PipeLLMTestCases:
system_prompt="You are a data analyst",
prompt="Analyze: @data",
output="Analysis",
- model=LLMSkill.LLM_TO_ANALYZE_DATA,
+ model=LLMSkill.LLM_TO_ANSWER_QUESTIONS,
),
)
@@ -138,7 +138,7 @@ class PipeLLMTestCases:
inputs={},
output="Item[]",
prompt="Generate items",
- llm_skill=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,
+ llm_skill=LLMSkill.LLM_FOR_WRITING_CHEAP,
),
PipeLLMBlueprint(
source=None,
@@ -146,7 +146,7 @@ class PipeLLMTestCases:
description="Generate multiple items",
output="Item[]",
prompt="Generate items",
- model=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,
+ model=LLMSkill.LLM_FOR_WRITING_CHEAP,
),
)
@@ -158,7 +158,7 @@ class PipeLLMTestCases:
inputs={},
output="Item[5]",
prompt="Generate items",
- llm_skill=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,
+ llm_skill=LLMSkill.LLM_FOR_WRITING_CHEAP,
),
PipeLLMBlueprint(
source=None,
@@ -166,7 +166,7 @@ class PipeLLMTestCases:
description="Generate exactly 5 items",
output="Item[5]",
prompt="Generate items",
- model=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,
+ model=LLMSkill.LLM_FOR_WRITING_CHEAP,
),
)
diff --git a/tests/unit/cli/test_init_cmd.py b/tests/unit/cli/test_init_cmd.py
index 513a30594..a6d91dde6 100644
--- a/tests/unit/cli/test_init_cmd.py
+++ b/tests/unit/cli/test_init_cmd.py
@@ -10,7 +10,7 @@ if TYPE_CHECKING:
from pytest_mock import MockerFixture
-from pipelex.cli.commands.init_cmd import init_config
+from pipelex.cli.commands.init.config_files import init_config
from pipelex.exceptions import PipelexCLIError
@@ -25,11 +25,11 @@ class TestInitCmd:
target_dir.mkdir()
# Mock get_configs_dir and config manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
mock_echo = mocker.patch("typer.echo")
# Execute
@@ -51,11 +51,11 @@ class TestInitCmd:
(target_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '0.9'")
# Mock get_configs_dir and config manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
mock_echo = mocker.patch("typer.echo")
# Execute
@@ -84,11 +84,11 @@ class TestInitCmd:
(target_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '0.9'")
# Mock get_configs_dir and config manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
mock_echo = mocker.patch("typer.echo")
# Execute
@@ -121,11 +121,11 @@ class TestInitCmd:
target_dir.mkdir()
# Mock get_configs_dir and config manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
mock_echo = mocker.patch("typer.echo")
# Execute
@@ -152,11 +152,11 @@ class TestInitCmd:
target_dir.mkdir()
# Mock get_configs_dir and config manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
mocker.patch("shutil.copy2", side_effect=PermissionError("Permission denied"))
# Execute and verify exception
@@ -174,11 +174,11 @@ class TestInitCmd:
target_dir = tmp_path / ".pipelex" # Don't create this
# Mock get_configs_dir and config manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
mock_echo = mocker.patch("typer.echo")
# Execute
@@ -209,10 +209,10 @@ class TestInitCmd:
target_dir.mkdir()
# Mock config_manager
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
# Execute dry-run
count_dry = init_config(reset=False, dry_run=True)
@@ -236,12 +236,12 @@ class TestInitCmd:
target_dir.mkdir()
# Mock config_manager and customize function
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
- mock_customize = mocker.patch("pipelex.cli.commands.init_cmd.customize_backends_config")
+ mock_customize = mocker.patch("pipelex.cli.commands.init.backends.customize_backends_config")
mocker.patch("typer.echo")
# Execute
@@ -261,12 +261,12 @@ class TestInitCmd:
(target_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '0.9'")
# Mock config_manager and customize function
- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)
+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)
mock_config_manager = mocker.MagicMock()
mock_config_manager.pipelex_config_dir = str(target_dir)
- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)
+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)
- mock_customize = mocker.patch("pipelex.cli.commands.init_cmd.customize_backends_config")
+ mock_customize = mocker.patch("pipelex.cli.commands.init.backends.customize_backends_config")
mocker.patch("typer.echo")
# Execute
diff --git a/tests/unit/cogt/model_routing/__init__.py b/tests/unit/cogt/model_routing/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/cogt/model_routing/test_routing_profile_fallback.py b/tests/unit/cogt/model_routing/test_routing_profile_fallback.py
new file mode 100644
index 000000000..0ec9fb61c
--- /dev/null
+++ b/tests/unit/cogt/model_routing/test_routing_profile_fallback.py
@@ -0,0 +1,272 @@
+from pipelex.cogt.model_routing.routing_models import BackendMatchingMethod
+from pipelex.cogt.model_routing.routing_profile import RoutingProfile
+
+
+class TestRoutingProfileFallbackOrder:
+ """Tests for the fallback_order functionality in RoutingProfile."""
+
+ def test_fallback_order_used_when_default_not_set(self):
+ """Test that fallback_order[0] is used as primary backend when default is not set."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ fallback_order=["openai", "anthropic", "google"],
+ )
+ enabled_backends = ["openai", "anthropic", "google"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.backend_name == "openai"
+ assert result.matching_method == BackendMatchingMethod.DEFAULT
+ assert result.fallback_order == ["openai", "anthropic", "google"]
+
+ def test_default_used_when_both_default_and_fallback_order_set(self):
+ """Test that default is used as primary backend when both default and fallback_order are set."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ default="anthropic",
+ fallback_order=["openai", "google"],
+ )
+ enabled_backends = ["openai", "anthropic", "google"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.backend_name == "anthropic"
+ assert result.matching_method == BackendMatchingMethod.DEFAULT
+ assert result.fallback_order == ["openai", "google"]
+
+ def test_fallback_order_included_in_result_when_default_is_set(self):
+ """Test that fallback_order is included in result even when default is used."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ default="openai",
+ fallback_order=["anthropic", "google"],
+ )
+ enabled_backends = ["openai", "anthropic", "google"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.backend_name == "openai"
+ assert result.fallback_order == ["anthropic", "google"]
+
+ def test_nonexistent_backend_in_fallback_order_is_filtered_out(self):
+ """Test that nonexistent backends in fallback_order are silently filtered out."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ fallback_order=["openai", "nonexistent_backend", "google"],
+ )
+ enabled_backends = ["openai", "google"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert - nonexistent_backend should be filtered out
+ assert result is not None
+ assert result.backend_name == "openai" # First enabled backend
+ assert result.fallback_order == ["openai", "google"] # Only enabled backends
+
+ def test_disabled_backend_in_fallback_order_is_filtered_out(self):
+ """Test that disabled backends in fallback_order are silently filtered out."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ fallback_order=["openai", "anthropic", "google"],
+ )
+ enabled_backends = ["openai", "google"] # anthropic is disabled
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert - anthropic should be filtered out
+ assert result is not None
+ assert result.backend_name == "openai"
+ assert result.fallback_order == ["openai", "google"] # Only enabled backends
+
+ def test_current_behavior_preserved_when_neither_default_nor_fallback_order_set(self):
+ """Test that None is returned when neither default nor fallback_order is set."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ )
+ enabled_backends = ["openai", "anthropic", "google"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is None
+
+ def test_exact_match_takes_precedence_over_fallback_order(self):
+ """Test that exact route match takes precedence over fallback_order."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ routes={"test-model": "google"},
+ fallback_order=["openai", "anthropic"],
+ )
+ enabled_backends = ["openai", "anthropic", "google"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.backend_name == "google"
+ assert result.matching_method == BackendMatchingMethod.EXACT_MATCH
+ assert result.fallback_order is None # Not DEFAULT match, so no fallback_order
+
+ def test_pattern_match_takes_precedence_over_fallback_order(self):
+ """Test that pattern match takes precedence over fallback_order."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ routes={"gpt-*": "openai"},
+ fallback_order=["anthropic", "google"],
+ )
+ enabled_backends = ["openai", "anthropic", "google"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="gpt-4",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.backend_name == "openai"
+ assert result.matching_method == BackendMatchingMethod.PATTERN_MATCH
+ assert result.fallback_order is None # Not DEFAULT match, so no fallback_order
+
+ def test_fallback_order_with_single_backend(self):
+ """Test that fallback_order works with a single backend."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ fallback_order=["openai"],
+ )
+ enabled_backends = ["openai", "anthropic"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.backend_name == "openai"
+ assert result.fallback_order == ["openai"]
+
+ def test_empty_fallback_order_treated_as_none(self):
+ """Test that empty fallback_order is treated as if not set."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ fallback_order=[],
+ )
+ enabled_backends = ["openai", "anthropic"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is None
+
+ def test_default_not_in_enabled_backends_falls_back_to_fallback_order(self):
+ """Test that if default is not in enabled_backends, fallback_order is used."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ default="disabled_backend",
+ fallback_order=["openai", "anthropic"],
+ )
+ enabled_backends = ["openai", "anthropic"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.backend_name == "openai" # First from fallback_order
+ assert result.fallback_order == ["openai", "anthropic"]
+
+ def test_fallback_order_filters_disabled_backends(self):
+ """Test that disabled backends are filtered out from fallback_order."""
+ # Arrange
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ fallback_order=["backend1", "backend2", "backend3"],
+ )
+ enabled_backends = ["backend1"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert - only enabled backends should remain
+ assert result is not None
+ assert result.backend_name == "backend1"
+ assert result.fallback_order == ["backend1"] # backend2 and backend3 filtered out
+
+ def test_fallback_order_preserved_in_result(self):
+ """Test that the exact fallback_order list is preserved in the result."""
+ # Arrange
+ fallback_order = ["openai", "anthropic", "google", "mistral"]
+ routing_profile = RoutingProfile(
+ name="test_profile",
+ default="openai",
+ fallback_order=fallback_order,
+ )
+ enabled_backends = ["openai", "anthropic", "google", "mistral"]
+
+ # Act
+ result = routing_profile.get_backend_match_for_model(
+ enabled_backends=enabled_backends,
+ model_name="test-model",
+ )
+
+ # Assert
+ assert result is not None
+ assert result.fallback_order == fallback_order
+ assert result.fallback_order is not fallback_order # Should be a copy/separate reference
diff --git a/tests/unit/cogt/models/test_model_deck.py b/tests/unit/cogt/models/test_model_deck.py
index 6b928f8e6..8b317663f 100644
--- a/tests/unit/cogt/models/test_model_deck.py
+++ b/tests/unit/cogt/models/test_model_deck.py
@@ -1,10 +1,13 @@
import pytest
+from pipelex.cogt.config_cogt import ModelDeckConfig
+from pipelex.cogt.exceptions import ModelWaterfallError
from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaults
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.models.model_deck import ModelDeck
from pipelex.cogt.usage.cost_category import CostCategory
+from pipelex.system.runtime import ProblemReaction
class TestModelDeckGetOptionalInferenceModel:
@@ -23,11 +26,14 @@ class TestModelDeckGetOptionalInferenceModel:
def _create_test_model_deck(
self,
inference_models: dict[str, InferenceModelSpec] | None = None,
- aliases: dict[str, str | list[str]] | None = None,
+ aliases: dict[str, str] | None = None,
+ fallbacks: dict[str, list[str]] | None = None,
+ is_model_fallback_enabled: bool = False,
) -> ModelDeck:
return ModelDeck(
inference_models=inference_models or {},
aliases=aliases or {},
+ waterfalls=fallbacks or {},
llm_presets={},
llm_choice_defaults=LLMSettingChoicesDefaults(
for_text=LLMSetting(model="default_text", temperature=0.7, max_tokens=1000),
@@ -37,6 +43,7 @@ class TestModelDeckGetOptionalInferenceModel:
extract_choice_default="extract_text_from_visuals",
img_gen_presets={},
img_gen_choice_default="gen_image_basic",
+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=is_model_fallback_enabled, missing_presets_reaction=ProblemReaction.NONE),
)
def test_direct_model_lookup_success(self):
@@ -84,10 +91,14 @@ class TestModelDeckGetOptionalInferenceModel:
def test_list_alias_resolution_first_success(self):
# Arrange
model_spec = self._create_test_model_spec("gpt-4")
- model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": ["gpt-4", "claude-3"]})
+ model_deck = self._create_test_model_deck(
+ inference_models={"gpt-4": model_spec},
+ fallbacks={"dummy-model-handle": ["gpt-4", "claude-3"]},
+ is_model_fallback_enabled=True,
+ )
# Act
- result = model_deck.get_optional_inference_model("best-model")
+ result = model_deck.get_optional_inference_model("dummy-model-handle")
# Assert
assert result == model_spec
@@ -97,32 +108,39 @@ class TestModelDeckGetOptionalInferenceModel:
model_spec = self._create_test_model_spec("claude-3")
model_deck = self._create_test_model_deck(
inference_models={"claude-3": model_spec},
- aliases={"best-model": ["nonexistent-model", "claude-3"]},
+ fallbacks={"dummy-model-handle": ["nonexistent-model", "claude-3"]},
+ is_model_fallback_enabled=True,
)
# Act
- result = model_deck.get_optional_inference_model("best-model")
+ result = model_deck.get_optional_inference_model("dummy-model-handle")
# Assert
assert result == model_spec
def test_list_alias_resolution_none_found(self):
# Arrange
- model_deck = self._create_test_model_deck(aliases={"best-model": ["nonexistent-1", "nonexistent-2"]})
-
- # Act
- result = model_deck.get_optional_inference_model("best-model")
+ model_deck = self._create_test_model_deck(
+ fallbacks={"dummy-model-handle": ["nonexistent-1", "nonexistent-2"]},
+ is_model_fallback_enabled=True,
+ )
- # Assert
- assert result is None
+ # Act & Assert
+ with pytest.raises(
+ ModelWaterfallError,
+ match=r"is a waterfall.*but none of the fallback models were found",
+ ):
+ model_deck.get_optional_inference_model("dummy-model-handle")
def test_recursive_alias_resolution_success(self):
# Arrange
model_spec = self._create_test_model_spec("gpt-4")
- model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": "best-gpt", "best-gpt": "gpt-4"})
+ model_deck = self._create_test_model_deck(
+ inference_models={"gpt-4": model_spec}, aliases={"dummy-model-handle": "best-gpt", "best-gpt": "gpt-4"}
+ )
# Act
- result = model_deck.get_optional_inference_model("best-model")
+ result = model_deck.get_optional_inference_model("dummy-model-handle")
# Assert
assert result == model_spec
@@ -132,18 +150,23 @@ class TestModelDeckGetOptionalInferenceModel:
model_spec = self._create_test_model_spec("gpt-4")
model_deck = self._create_test_model_deck(
inference_models={"gpt-4": model_spec},
- aliases={"best-model": ["nonexistent", "best-gpt"], "best-gpt": "gpt-4"},
+ aliases={"best-gpt": "gpt-4"},
+ fallbacks={"dummy-model-handle": ["nonexistent", "best-gpt"]},
+ is_model_fallback_enabled=True,
)
# Act
- result = model_deck.get_optional_inference_model("best-model")
+ result = model_deck.get_optional_inference_model("dummy-model-handle")
# Assert
assert result == model_spec
def test_empty_alias_list(self):
# Arrange
- model_deck = self._create_test_model_deck(aliases={"empty-alias": []})
+ model_deck = self._create_test_model_deck(
+ fallbacks={"empty-alias": []},
+ is_model_fallback_enabled=True,
+ )
# Act
result = model_deck.get_optional_inference_model("empty-alias")
@@ -172,11 +195,13 @@ class TestModelDeckGetOptionalInferenceModel:
model_spec = self._create_test_model_spec("claude-3")
model_deck = self._create_test_model_deck(
inference_models={"claude-3": model_spec},
- aliases={"best-model": ["premium-gpt", "premium-claude"], "premium-gpt": ["gpt-4-turbo", "gpt-4"], "premium-claude": "claude-3"},
+ aliases={"premium-claude": "claude-3"},
+ fallbacks={"dummy-model-handle": ["premium-gpt", "premium-claude"]},
+ is_model_fallback_enabled=True,
)
# Act
- result = model_deck.get_optional_inference_model("best-model")
+ result = model_deck.get_optional_inference_model("dummy-model-handle")
# Assert
assert result == model_spec
@@ -189,9 +214,12 @@ class TestModelDeckGetOptionalInferenceModel:
inference_models={"gpt-4": model_spec1, "claude-3": model_spec2},
aliases={
"ai-model": "best-gpt", # string alias
- "best-gpt": ["gpt-4-turbo", "gpt-4"], # list alias
- "backup-model": ["claude-4", "claude-3"], # list alias
},
+ fallbacks={
+ "best-gpt": ["gpt-4-turbo", "gpt-4"], # list fallback
+ "backup-model": ["claude-4", "claude-3"], # list fallback
+ },
+ is_model_fallback_enabled=True,
)
# Act
diff --git a/tests/unit/kit/test_paths.py b/tests/unit/kit/test_paths.py
index 4713cc79b..fe785ba1d 100644
--- a/tests/unit/kit/test_paths.py
+++ b/tests/unit/kit/test_paths.py
@@ -1,4 +1,4 @@
-from pipelex.kit.paths import get_agents_dir, get_configs_dir, get_kit_root
+from pipelex.kit.paths import get_kit_agents_dir, get_kit_configs_dir, get_kit_root
class TestKitPaths:
@@ -12,12 +12,12 @@ class TestKitPaths:
def test_get_agents_dir(self):
"""Test that agents directory path is valid."""
- agents_dir = get_agents_dir()
+ agents_dir = get_kit_agents_dir()
assert agents_dir is not None
assert agents_dir.is_dir()
def test_get_configs_dir(self):
"""Test that configs directory path is valid."""
- configs_dir = get_configs_dir()
+ configs_dir = get_kit_configs_dir()
assert configs_dir is not None
assert configs_dir.is_dir()
diff --git a/tests/unit/tools/config/test_load_config.py b/tests/unit/tools/config/test_load_config.py
index d0c297161..68a7d79d5 100644
--- a/tests/unit/tools/config/test_load_config.py
+++ b/tests/unit/tools/config/test_load_config.py
@@ -1,8 +1,17 @@
from pipelex.config import PipelexConfig
-from pipelex.hub import get_pipelex_hub
+from pipelex.system.configuration.config_loader import config_manager
+from pipelex.tools.misc.json_utils import deep_update
+from pipelex.tools.misc.toml_utils import load_toml_from_path
class TestLoadConfig:
- def test_load_config(self):
- hub = get_pipelex_hub()
- hub.setup_config(config_cls=PipelexConfig, specific_config_path="pipelex/kit/configs/pipelex.toml")
+ def test_load_kit_config(self):
+ # The base condif must validate
+ config = config_manager.load_config()
+ PipelexConfig.model_validate(config)
+
+ # The confiug updated by the kit config must validate too
+ config_path = "pipelex/kit/configs/pipelex.toml"
+ kit_config = load_toml_from_path(config_path)
+ deep_update(config, kit_config)
+ PipelexConfig.model_validate(config)
diff --git a/tests/unit/tools/secrets/test_secrets_utils.py b/tests/unit/tools/secrets/test_secrets_utils.py
index 1cbe8339e..6794b0c20 100644
--- a/tests/unit/tools/secrets/test_secrets_utils.py
+++ b/tests/unit/tools/secrets/test_secrets_utils.py
@@ -16,44 +16,40 @@ def mock_secrets_provider(mocker: MockerFixture) -> Any:
class TestSubstituteVars:
- def test_simple_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_simple_secret_substitution(self, mock_secrets_provider: Any) -> None:
mock_secrets_provider.get_secret.return_value = "secret_value"
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
- result = substitute_vars("Hello ${API_KEY}")
+ result = substitute_vars("Hello ${API_KEY}", mock_secrets_provider)
assert result == "Hello secret_value"
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")
- def test_explicit_env_substitution(self, mocker: MockerFixture) -> None:
+ def test_explicit_env_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
mocker.patch.dict(os.environ, {"TEST_VAR": "env_value"})
- result = substitute_vars("Value: ${env:TEST_VAR}")
+ result = substitute_vars("Value: ${env:TEST_VAR}", mock_secrets_provider)
assert result == "Value: env_value"
- def test_explicit_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_explicit_secret_substitution(self, mock_secrets_provider: Any) -> None:
mock_secrets_provider.get_secret.return_value = "secret_value"
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
- result = substitute_vars("Value: ${secret:API_KEY}")
+ result = substitute_vars("Value: ${secret:API_KEY}", mock_secrets_provider)
assert result == "Value: secret_value"
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")
- def test_fallback_env_to_secret(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_fallback_env_to_secret(self, mock_secrets_provider: Any) -> None:
mock_secrets_provider.get_secret.return_value = "secret_fallback"
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
- result = substitute_vars("Value: ${env:MISSING_VAR|secret:FALLBACK_KEY}")
+ result = substitute_vars("Value: ${env:MISSING_VAR|secret:FALLBACK_KEY}", mock_secrets_provider)
assert result == "Value: secret_fallback"
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="FALLBACK_KEY")
def test_env_takes_precedence_in_fallback(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
mocker.patch.dict(os.environ, {"EXISTING_VAR": "env_value"})
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
- result = substitute_vars("Value: ${env:EXISTING_VAR|secret:FALLBACK_KEY}")
+ result = substitute_vars("Value: ${env:EXISTING_VAR|secret:FALLBACK_KEY}", mock_secrets_provider)
assert result == "Value: env_value"
# Secret should not be called since env var was found
@@ -63,9 +59,8 @@ class TestSubstituteVars:
"""Test fallback from secret to env when secret doesn't exist."""
mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")
mocker.patch.dict(os.environ, {"FALLBACK_ENV": "env_fallback"})
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
- result = substitute_vars("Value: ${secret:MISSING_SECRET|env:FALLBACK_ENV}")
+ result = substitute_vars("Value: ${secret:MISSING_SECRET|env:FALLBACK_ENV}", mock_secrets_provider)
assert result == "Value: env_fallback"
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="MISSING_SECRET")
@@ -74,9 +69,8 @@ class TestSubstituteVars:
"""Test that secret takes precedence when both secret and env are available in reverse order."""
mock_secrets_provider.get_secret.return_value = "secret_value"
mocker.patch.dict(os.environ, {"EXISTING_VAR": "env_value"})
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
- result = substitute_vars("Value: ${secret:EXISTING_SECRET|env:EXISTING_VAR}")
+ result = substitute_vars("Value: ${secret:EXISTING_SECRET|env:EXISTING_VAR}", mock_secrets_provider)
assert result == "Value: secret_value"
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="EXISTING_SECRET")
@@ -85,85 +79,77 @@ class TestSubstituteVars:
"""Test multiple variable substitutions in the same content."""
mock_secrets_provider.get_secret.side_effect = lambda secret_id: f"secret_{secret_id}" # pyright: ignore[reportUnknownLambdaType]
mocker.patch.dict(os.environ, {"ENV_VAR": "env_value"})
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
content = "API: ${API_KEY}, DB: ${env:ENV_VAR}, Token: ${secret:TOKEN}"
- result = substitute_vars(content)
+ result = substitute_vars(content, mock_secrets_provider)
assert result == "API: secret_API_KEY, DB: env_value, Token: secret_TOKEN"
- def test_no_placeholders(self) -> None:
+ def test_no_placeholders(self, mock_secrets_provider: Any) -> None:
"""Test content with no placeholders remains unchanged."""
content = "This is just plain text with no variables"
- result = substitute_vars(content)
+ result = substitute_vars(content, mock_secrets_provider)
assert result == content
- def test_missing_env_var_raises_error(self) -> None:
+ def test_missing_env_var_raises_error(self, mock_secrets_provider: Any) -> None:
"""Test that missing required env var raises VarNotFoundError."""
with pytest.raises(VarNotFoundError, match="Environment variable 'MISSING_VAR' is required but not set"):
- substitute_vars("Value: ${env:MISSING_VAR}")
+ substitute_vars("Value: ${env:MISSING_VAR}", mock_secrets_provider)
- def test_missing_secret_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_missing_secret_raises_error(self, mock_secrets_provider: Any) -> None:
"""Test that missing required secret raises error."""
mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
with pytest.raises(VarNotFoundError, match="Could not get variable 'MISSING_SECRET': Secret not found"):
- substitute_vars("Value: ${secret:MISSING_SECRET}")
+ substitute_vars("Value: ${secret:MISSING_SECRET}", mock_secrets_provider)
- def test_missing_default_secret_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_missing_default_secret_raises_error(self, mock_secrets_provider: Any) -> None:
"""Test that missing default secret (no prefix) raises error."""
mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
with pytest.raises(VarNotFoundError, match="Could not get variable 'MISSING_SECRET': Secret not found"):
- substitute_vars("Value: ${MISSING_SECRET}")
+ substitute_vars("Value: ${MISSING_SECRET}", mock_secrets_provider)
- def test_fallback_both_missing_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_fallback_both_missing_raises_error(self, mock_secrets_provider: Any) -> None:
mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
with pytest.raises(VarFallbackPatternError, match="Could not get variable from fallback pattern: env:MISSING_ENV\\|secret:MISSING_SECRET"):
- substitute_vars("Value: ${env:MISSING_ENV|secret:MISSING_SECRET}")
+ substitute_vars("Value: ${env:MISSING_ENV|secret:MISSING_SECRET}", mock_secrets_provider)
- def test_reverse_fallback_both_missing_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_reverse_fallback_both_missing_raises_error(self, mock_secrets_provider: Any) -> None:
"""Test that when both secret and env are missing in reverse order, error is raised."""
mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
with pytest.raises(VarFallbackPatternError, match="Could not get variable from fallback pattern: secret:MISSING_SECRET\\|env:MISSING_ENV"):
- substitute_vars("Value: ${secret:MISSING_SECRET|env:MISSING_ENV}")
+ substitute_vars("Value: ${secret:MISSING_SECRET|env:MISSING_ENV}", mock_secrets_provider)
def test_complex_variable_names(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
"""Test complex variable names with underscores, numbers, etc."""
mock_secrets_provider.get_secret.return_value = "complex_secret"
mocker.patch.dict(os.environ, {"API_KEY_V2": "complex_env"})
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
- result = substitute_vars("${env:API_KEY_V2} and ${DB_PASSWORD_123}")
+ result = substitute_vars("${env:API_KEY_V2} and ${DB_PASSWORD_123}", mock_secrets_provider)
assert result == "complex_env and complex_secret"
- def test_whitespace_handling(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_whitespace_handling(self, mock_secrets_provider: Any) -> None:
"""Test that whitespace around variable names is handled correctly."""
mock_secrets_provider.get_secret.return_value = "secret_value"
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
# Note: whitespace should be preserved as part of the variable name
- result = substitute_vars("Value: ${ API_KEY }")
+ result = substitute_vars("Value: ${ API_KEY }", mock_secrets_provider)
assert result == "Value: secret_value"
mock_secrets_provider.get_secret.assert_called_once_with(secret_id=" API_KEY ")
- def test_nested_braces_not_matched(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
+ def test_nested_braces_not_matched(self, mock_secrets_provider: Any) -> None:
"""Test that nested braces are not matched by the pattern."""
# The content has nested braces that should not be matched as a single variable
# But ${nested} should be matched as a valid variable
mock_secrets_provider.get_secret.return_value = "secret_nested"
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
content = "This ${has ${nested} braces} should not match"
- result = substitute_vars(content)
+ result = substitute_vars(content, mock_secrets_provider)
# The ${nested} part should be substituted, but ${has ... should not be matched
# because it contains a $ character which breaks the pattern
@@ -174,34 +160,33 @@ class TestSubstituteVars:
"""Test substitution in multiline content."""
mock_secrets_provider.get_secret.return_value = "secret_value"
mocker.patch.dict(os.environ, {"HOME": "/home/user"})
- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
content = """Line 1: ${API_KEY}
Line 2: Some text
Line 3: ${env:HOME}"""
- result = substitute_vars(content)
+ result = substitute_vars(content, mock_secrets_provider)
expected = """Line 1: secret_value
Line 2: Some text
Line 3: /home/user"""
assert result == expected
- def test_pattern_does_not_match_across_newlines(self) -> None:
+ def test_pattern_does_not_match_across_newlines(self, mock_secrets_provider: Any) -> None:
"""Test that the pattern doesn't match variables that span multiple lines."""
content = """This ${VAR
NAME} should not match"""
- result = substitute_vars(content)
+ result = substitute_vars(content, mock_secrets_provider)
# Should remain unchanged
assert result == content
- def test_pattern_does_not_match_with_quotes(self) -> None:
+ def test_pattern_does_not_match_with_quotes(self, mock_secrets_provider: Any) -> None:
"""Test that the pattern doesn't match variables containing quotes."""
content = "This ${VAR\"NAME} and ${VAR'NAME} should not match"
- result = substitute_vars(content)
+ result = substitute_vars(content, mock_secrets_provider)
# Should remain unchanged
assert result == content
- def test_unknown_prefix_raises_error(self) -> None:
+ def test_unknown_prefix_raises_error(self, mock_secrets_provider: Any) -> None:
"""Test that unknown prefix raises UnknownVarPrefixError."""
with pytest.raises(UnknownVarPrefixError, match="Unknown variable prefix: 'foo'"):
- substitute_vars("Value: ${foo:BAR}")
+ substitute_vars("Value: ${foo:BAR}", mock_secrets_provider)
diff --git a/tests/unit/tools/typing/test_custom_base_model.py b/tests/unit/tools/typing/test_custom_base_model.py
new file mode 100644
index 000000000..e8f971855
--- /dev/null
+++ b/tests/unit/tools/typing/test_custom_base_model.py
@@ -0,0 +1,25 @@
+from pipelex.tools.misc.attribute_utils import AttributePolisher
+from pipelex.tools.typing.pydantic_utils import CustomBaseModel
+
+
+class TestCustomBaseModel:
+ """Tests for CustomBaseModel class."""
+
+ def test_custom_base_model_truncates_repr(self) -> None:
+ """Test that CustomBaseModel truncates long base64 and URL strings in repr output."""
+
+ class TestModel(CustomBaseModel):
+ base_64: str
+ url: str
+ other: str
+
+ AttributePolisher.base_64_truncate_length = 10
+ AttributePolisher.url_truncate_length = 10
+ model = TestModel(
+ base_64="b" * 20,
+ url="data:image/png;base64," + "x" * 20,
+ other="val",
+ )
+ repr_str = repr(model)
+ assert "bbbbbbbbbb…" in repr_str
+ assert "data:image…" in repr_str
diff --git a/tests/unit/tools/typing/test_enum_utilities.py b/tests/unit/tools/typing/test_enum_utilities.py
new file mode 100644
index 000000000..73f0c0a01
--- /dev/null
+++ b/tests/unit/tools/typing/test_enum_utilities.py
@@ -0,0 +1,47 @@
+from pydantic import BaseModel, Field
+
+from pipelex.tools.typing.pydantic_utils import (
+ ExtraFieldAttribute,
+ clean_model_to_dict,
+ convert_strenum_to_str,
+)
+from pipelex.types import StrEnum
+
+
+class SimpleEnum(StrEnum):
+ FIRST = "first"
+ SECOND = "second"
+
+ def display_name(self) -> str: # pragma: no cover - simple helper
+ return self.value.upper()
+
+
+class ModelWithEnum(BaseModel):
+ name: str
+ enum_val: SimpleEnum
+ secret: str = Field(
+ "hidden",
+ json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True},
+ )
+
+
+class TestEnumUtilities:
+ """Tests for enum utility functions."""
+
+ def test_clean_model_to_dict_stringifies_enums_and_hides_fields(self) -> None:
+ """Test that clean_model_to_dict converts enums to strings and hides marked fields."""
+ model = ModelWithEnum(name="foo", enum_val=SimpleEnum.FIRST, secret="hidden")
+ result = clean_model_to_dict(model)
+ assert result == {"name": "foo", "enum_val": "FIRST"}
+
+ def test_convert_strenum_to_str_recursive(self) -> None:
+ """Test that convert_strenum_to_str recursively converts enums in nested structures."""
+ data = {
+ "enum": SimpleEnum.SECOND,
+ "nested": [SimpleEnum.FIRST, {"e": SimpleEnum.SECOND}],
+ }
+ converted = convert_strenum_to_str(data)
+ assert converted == {
+ "enum": "SECOND",
+ "nested": ["FIRST", {"e": "SECOND"}],
+ }
diff --git a/tests/unit/tools/typing/test_format_pydantic_validation_error.py b/tests/unit/tools/typing/test_format_pydantic_validation_error.py
new file mode 100644
index 000000000..3548f733f
--- /dev/null
+++ b/tests/unit/tools/typing/test_format_pydantic_validation_error.py
@@ -0,0 +1,98 @@
+import pytest
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+
+
+class TestFormatPydanticValidationError:
+ """Tests for format_pydantic_validation_error function."""
+
+ def test_single_missing_and_extra_field(self) -> None:
+ """Test formatting with a single missing and extra field."""
+
+ class Validated(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ a: int
+ b: str
+
+ with pytest.raises(ValidationError) as exc:
+ Validated.model_validate({"a": "not_int", "c": 1})
+
+ formatted = format_pydantic_validation_error(exc.value)
+ assert "Missing required fields: 'b'" in formatted
+ assert "Extra forbidden fields: 'c'" in formatted
+
+ def test_multiple_missing_fields(self) -> None:
+ """Test formatting with multiple missing required fields displayed as comma-separated list."""
+
+ class MultiRequired(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ field1: str
+ field2: int
+ field3: bool
+
+ with pytest.raises(ValidationError) as exc:
+ MultiRequired.model_validate({})
+
+ formatted = format_pydantic_validation_error(exc.value)
+ assert "Missing required fields:" in formatted
+ assert "'field1'" in formatted
+ assert "'field2'" in formatted
+ assert "'field3'" in formatted
+ # Check comma separation (order may vary)
+ assert ", " in formatted
+
+ def test_multiple_extra_fields(self) -> None:
+ """Test formatting with multiple extra forbidden fields displayed as comma-separated list."""
+
+ class NoExtras(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ valid_field: str
+
+ with pytest.raises(ValidationError) as exc:
+ NoExtras.model_validate({"valid_field": "ok", "extra1": 1, "extra2": 2, "extra3": 3})
+
+ formatted = format_pydantic_validation_error(exc.value)
+ assert "Extra forbidden fields:" in formatted
+ assert "'extra1'" in formatted
+ assert "'extra2'" in formatted
+ assert "'extra3'" in formatted
+ # Check comma separation
+ assert ", " in formatted
+
+ def test_combination_of_multiple_missing_and_extra_fields(self) -> None:
+ """Test formatting with both multiple missing and extra fields together."""
+
+ class MultiRequired(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ field1: str
+ field2: int
+ field3: bool
+
+ with pytest.raises(ValidationError) as exc:
+ MultiRequired.model_validate({"wrong1": "a", "wrong2": "b"})
+
+ formatted = format_pydantic_validation_error(exc.value)
+ assert "Missing required fields:" in formatted
+ assert "Extra forbidden fields:" in formatted
+ # Check for at least one missing field
+ assert "'field1'" in formatted or "'field2'" in formatted or "'field3'" in formatted
+ # Check for extra fields
+ assert "'wrong1'" in formatted
+ assert "'wrong2'" in formatted
+
+ def test_model_type_errors(self) -> None:
+ """Test formatting with model type validation errors."""
+
+ class InnerModel(BaseModel):
+ value: int
+
+ class OuterModel(BaseModel):
+ inner: InnerModel
+
+ with pytest.raises(ValidationError) as exc:
+ OuterModel.model_validate({"inner": "not_a_model"})
+
+ formatted = format_pydantic_validation_error(exc.value)
+ assert "Model type errors:" in formatted
+ assert "expected InnerModel, got str" in formatted
diff --git a/tests/unit/tools/typing/test_pydantic_utils.py b/tests/unit/tools/typing/test_pydantic_utils.py
deleted file mode 100644
index 26b010083..000000000
--- a/tests/unit/tools/typing/test_pydantic_utils.py
+++ /dev/null
@@ -1,129 +0,0 @@
-from typing import Any, cast
-
-import pytest
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
-
-from pipelex.tools.misc.attribute_utils import AttributePolisher
-from pipelex.tools.typing.pydantic_utils import (
- CustomBaseModel,
- ExtraFieldAttribute,
- clean_model_to_dict,
- convert_strenum_to_str,
- format_pydantic_validation_error,
- serialize_model,
-)
-from pipelex.types import StrEnum
-
-
-class ChildModel(BaseModel):
- child_name: str
- child_secret: str = Field(
- ...,
- json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True}, # This field should be hidden
- )
-
-
-class ParentModel(BaseModel):
- parent_name: str
- child: ChildModel
-
-
-def test_serialize_model_with_hidden_fields():
- # Create nested models
- child = ChildModel(child_name="foo", child_secret="secret_foo")
- parent = ParentModel(parent_name="bar", child=child)
-
- # Test default model_dump() shows everything
- model_dump_result: dict[str, Any] = parent.model_dump()
- child_dict = cast("dict[str, Any]", model_dump_result["child"])
- assert "child_secret" in child_dict
- assert child_dict["child_secret"] == "secret_foo"
-
- # Test our custom serializer omits hidden fields (child_secret)
- serialized_result = cast("dict[str, Any]", serialize_model(parent))
- child_serialized = cast("dict[str, Any]", serialized_result["child"])
- assert "child_secret" not in child_serialized
- assert serialized_result == {"parent_name": "bar", "child": {"child_name": "foo"}}
-
-
-class SimpleEnum(StrEnum):
- FIRST = "first"
- SECOND = "second"
-
- def display_name(self) -> str: # pragma: no cover - simple helper
- return self.value.upper()
-
-
-class ModelWithEnum(BaseModel):
- name: str
- enum_val: SimpleEnum
- secret: str = Field(
- "hidden",
- json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True},
- )
-
-
-def test_clean_model_to_dict_stringifies_enums_and_hides_fields() -> None:
- model = ModelWithEnum(name="foo", enum_val=SimpleEnum.FIRST, secret="hidden")
- result = clean_model_to_dict(model)
- assert result == {"name": "foo", "enum_val": "FIRST"}
-
-
-def test_convert_strenum_to_str_recursive() -> None:
- data = {
- "enum": SimpleEnum.SECOND,
- "nested": [SimpleEnum.FIRST, {"e": SimpleEnum.SECOND}],
- }
- converted = convert_strenum_to_str(data)
- assert converted == {
- "enum": "SECOND",
- "nested": ["FIRST", {"e": "SECOND"}],
- }
-
-
-def test_format_pydantic_validation_error() -> None:
- class Validated(BaseModel):
- model_config = ConfigDict(extra="forbid")
- a: int
- b: str
-
- with pytest.raises(ValidationError) as exc:
- Validated.model_validate({"a": "not_int", "c": 1})
-
- formatted = format_pydantic_validation_error(exc.value)
- assert "Missing required fields: ['b']" in formatted
- assert "Extra forbidden fields: ['c: 1']" in formatted
-
-
-def test_format_pydantic_validation_error_model_type() -> None:
- class InnerModel(BaseModel):
- value: int
-
- class OuterModel(BaseModel):
- inner: InnerModel
-
- # Try to pass a dict where a specific model type is expected
- with pytest.raises(ValidationError) as exc:
- OuterModel.model_validate({"inner": "not_a_model"})
-
- formatted = format_pydantic_validation_error(exc.value)
- assert "Model type errors:" in formatted
- assert "expected InnerModel, got str" in formatted
-
-
-def test_custom_base_model_truncates_repr() -> None:
- class TestModel(CustomBaseModel):
- base_64: str
- url: str
- other: str
-
- AttributePolisher.base_64_truncate_length = 10
- AttributePolisher.url_truncate_length = 10
- model = TestModel(
- base_64="b" * 20,
- url="data:image/png;base64," + "x" * 20,
- other="val",
- )
- repr_str = repr(model)
- assert "bbbbbbbbbb…" in repr_str
- assert "data:image…" in repr_str
diff --git a/tests/unit/tools/typing/test_serialize_model.py b/tests/unit/tools/typing/test_serialize_model.py
new file mode 100644
index 000000000..b8b1fbbf1
--- /dev/null
+++ b/tests/unit/tools/typing/test_serialize_model.py
@@ -0,0 +1,40 @@
+from typing import Any, cast
+
+from pydantic import BaseModel, Field
+
+from pipelex.tools.typing.pydantic_utils import ExtraFieldAttribute, serialize_model
+
+
+class ChildModel(BaseModel):
+ child_name: str
+ child_secret: str = Field(
+ ...,
+ json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True}, # This field should be hidden
+ )
+
+
+class ParentModel(BaseModel):
+ parent_name: str
+ child: ChildModel
+
+
+class TestSerializeModel:
+ """Tests for serialize_model function."""
+
+ def test_serialize_model_with_hidden_fields(self) -> None:
+ """Test that serialize_model omits fields marked as hidden while model_dump includes them."""
+ # Create nested models
+ child = ChildModel(child_name="foo", child_secret="secret_foo")
+ parent = ParentModel(parent_name="bar", child=child)
+
+ # Test default model_dump() shows everything
+ model_dump_result: dict[str, Any] = parent.model_dump()
+ child_dict = cast("dict[str, Any]", model_dump_result["child"])
+ assert "child_secret" in child_dict
+ assert child_dict["child_secret"] == "secret_foo"
+
+ # Test our custom serializer omits hidden fields (child_secret)
+ serialized_result = cast("dict[str, Any]", serialize_model(parent))
+ child_serialized = cast("dict[str, Any]", serialized_result["child"])
+ assert "child_secret" not in child_serialized
+ assert serialized_result == {"parent_name": "bar", "child": {"child_name": "foo"}}