Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Feature/merge model fallback (#431)
Louis Choquel918f52f646ed
diff --git a/.badges/tests.json b/.badges/tests.jsonindex 3c153507c..2440a5710 100644--- a/.badges/tests.json+++ b/.badges/tests.json@@ -1,7 +1,7 @@{"schemaVersion": 1,"label": "tests",- "message": "1384",+ "message": "1434","color": "blue","cacheSeconds": 300}\ No newline at end of filediff --git a/.blackboxrules b/.blackboxrulesindex 23c6f20aa..6e236d028 100644--- a/.blackboxrules+++ b/.blackboxrules@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/.cursor/rules/llms.mdc b/.cursor/rules/llms.mdcindex 8babc7660..7622b337d 100644--- a/.cursor/rules/llms.mdc+++ b/.cursor/rules/llms.mdc@@ -55,7 +55,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/.cursor/rules/write_pipelex.mdc b/.cursor/rules/write_pipelex.mdcindex c64d617ee..b46e4f23d 100644--- a/.cursor/rules/write_pipelex.mdc+++ b/.cursor/rules/write_pipelex.mdc@@ -824,7 +824,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.mdindex 23c6f20aa..6e236d028 100644--- a/.github/copilot-instructions.md+++ b/.github/copilot-instructions.md@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/.pipelex/inference/backends/anthropic.toml b/.pipelex/inference/backends/anthropic.tomlindex 4b57964c4..66721e2c1 100644--- a/.pipelex/inference/backends/anthropic.toml+++ b/.pipelex/inference/backends/anthropic.toml@@ -89,3 +89,11 @@ inputs = ["text", "images"]outputs = ["text", "structured"]max_prompt_images = 100costs = { input = 3.0, output = 15.0 }++["claude-4.5-haiku"]+model_id = "claude-haiku-4-5-20251001"+max_tokens = 64000+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 100+costs = { input = 1.0, output = 5.0 }diff --git a/.pipelex/inference/backends/azure_openai.toml b/.pipelex/inference/backends/azure_openai.tomlindex 135638ad5..930f161fd 100644--- a/.pipelex/inference/backends/azure_openai.toml+++ b/.pipelex/inference/backends/azure_openai.toml@@ -80,6 +80,12 @@ inputs = ["text"]outputs = ["text", "structured"]costs = { input = 1.1, output = 4.4 }+[o3]+model_id = "o3-2025-04-16"+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 2, output = 8 }+# --- GPT-5 Series -------------------------------------------------------------[gpt-5-mini]model_id = "gpt-5-mini-2025-08-07"diff --git a/.pipelex/inference/backends/bedrock.toml b/.pipelex/inference/backends/bedrock.tomlindex c27baf0b5..cea91344d 100644--- a/.pipelex/inference/backends/bedrock.toml+++ b/.pipelex/inference/backends/bedrock.toml@@ -92,8 +92,6 @@ outputs = ["text", "structured"]max_prompt_images = 100costs = { input = 3.0, output = 15.0 }-# anthropic.claude-sonnet-4-5-20250929-v1:0,Anthropic,arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0,-["claude-4.5-sonnet"]sdk = "bedrock_anthropic"model_id = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"@@ -102,3 +100,12 @@ inputs = ["text", "images"]outputs = ["text", "structured"]max_prompt_images = 100costs = { input = 3.0, output = 15.0 }++["claude-4.5-haiku"]+sdk = "bedrock_anthropic"+model_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0"+max_tokens = 8192+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 100+costs = { input = 1.0, output = 5.0 }diff --git a/.pipelex/inference/backends/pipelex_inference.toml b/.pipelex/inference/backends/pipelex_inference.tomlindex 805b761ad..e516e274e 100644--- a/.pipelex/inference/backends/pipelex_inference.toml+++ b/.pipelex/inference/backends/pipelex_inference.toml@@ -103,6 +103,12 @@ inputs = ["text", "images"]outputs = ["text", "structured"]costs = { input = 3, output = 15 }+["claude-4.5-haiku"]+model_id = "pipelex/claude-4.5-haiku"+inputs = ["text", "images"]+outputs = ["text", "structured"]+costs = { input = 1, output = 5 }+# --- Gemini LLMs --------------------------------------------------------------["gemini-2.0-flash"]model_id = "gemini/gemini-2.0-flash"diff --git a/.pipelex/inference/deck/base_deck.toml b/.pipelex/inference/deck/base_deck.tomlindex 9b2d06f13..12f516755 100644--- a/.pipelex/inference/deck/base_deck.toml+++ b/.pipelex/inference/deck/base_deck.toml@@ -28,27 +28,59 @@ best-gemini = "gemini-2.5-pro"best-mistral = "mistral-medium"best-grok = "grok-3"-cheap-gpt = "gpt-4o-mini"-cheap_llm_for_text = "gpt-4o-mini"-cheap_llm_for_object = "gpt-4o-mini"-cheap_llm_for_vision = "gemini-2.5-flash-lite"+# Image generation aliases+base-img-gen = "flux-pro/v1.1"+best-img-gen = "flux-pro/v1.1-ultra"+fast-img-gen = "fast-lightning-sdxl"-base_llm_for_text = "gpt-4o-mini"-base_llm_for_object = "gpt-4o-mini"+####################################################################################################+# Waterfalls+####################################################################################################+[waterfalls]smart_llm = ["claude-4.5-sonnet","claude-4.1-opus","claude-4-sonnet","gemini-2.5-pro",- "gpt-5",+ "gpt-4o", # we're using gpt-4o here rather than gpt-5 until we'll handle reasoning parameters+ "grok-3",+]+smart_llm_with_vision = [+ "claude-4.5-sonnet",+ "claude-4.1-opus",+ "claude-4-sonnet",+ "gemini-2.5-pro",+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters+ "grok-3",+]+smart_llm_for_structured = [+ "claude-4.5-sonnet",+ "claude-4.1-opus",+ "claude-4-sonnet",+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters]-llm_for_large_codebase = ["gemini-2.5-pro", "claude-4.5-sonnet", "gpt-5"]-# Image generation aliases-base-img-gen = "flux-pro/v1.1-ultra"-best-img-gen = "nano-banana"-fast-img-gen = "fast-lightning-sdxl"+llm_for_large_codebase = [+ "gemini-2.5-pro",+ "claude-4.5-sonnet",+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters+ "gemini-2.5-flash",+ "grok-3",+]+cheap_llm = [+ "gpt-4o-mini",+ "gemini-2.5-flash-lite",+ "mistral-small",+ "claude-3-haiku",+ "grok-3-mini",+]+cheap_llm_for_vision = [+ "gpt-4o-mini",+ "gemini-2.5-flash-lite",+ "claude-3-haiku",+]+cheap_llm_for_structured = ["gpt-4o-mini", "mistral-small", "claude-3-haiku"]##################################################################################################### LLM Presets@@ -56,73 +88,47 @@ fast-img-gen = "fast-lightning-sdxl"[llm.presets]-####################################################################################################-# LLM Presets — General purpose--cheap_llm_for_text = { model = "cheap_llm_for_text", temperature = 0.5 }-cheap_llm_for_short_text = { model = "cheap_llm_for_text", temperature = 0.5, max_tokens = 50 }-cheap_llm_for_object = { model = "cheap_llm_for_object", temperature = 0.5 }-cheap_llm_to_structure = { model = "cheap_llm_for_object", temperature = 0.1 }--llm_for_testing_gen_text = { model = "base_llm_for_text", temperature = 0.5 }-llm_for_testing_gen_object = { model = "base_llm_for_object", temperature = 0.5 }-llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }-llm_for_testing_vision_structured = { model = "cheap-gpt", temperature = 0.5 }--####################################################################################################-# LLM Presets — Specific skills--# Cheap llm-llm_cheap = { model = "base-gemini", temperature = 0.1 }+# LLM Presets — Specific skills -------------------------------------------------------------# Generation skillsllm_for_factual_writing = { model = "base-gpt", temperature = 0.1 }llm_for_creative_writing = { model = "base-gpt", temperature = 0.9 }-llm_to_write_questions = { model = "gpt-4o-mini", temperature = 0.3 }--# Reasoning skills-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }-llm_to_reason_on_diagram = { model = "base-claude", temperature = 0.5 }-llm_to_analyze_data = { model = "base-claude", temperature = 0.5 }+llm_for_writing_cheap = { model = "gpt-4o-mini", temperature = 0.3 }-# Search and answer skills-llm_to_answer_easy_questions = { model = "gpt-4o-mini", temperature = 0.3 }-llm_to_answer_hard_questions = { model = "base-claude", temperature = 0.3 }+# Retrieve and answer questions skills+llm_to_answer_questions_cheap = { model = "gpt-4o-mini", temperature = 0.3 }+llm_to_answer_questions = { model = "base-claude", temperature = 0.3 }llm_to_retrieve = { model = "base-claude", temperature = 0.1 }-llm_for_enrichment = { model = "gpt-4o", temperature = 0.1 }-llm_to_enrich = { model = "base-claude", temperature = 0.1 }-llm_for_question_and_excerpt_reformulation = { model = "gpt-4o", temperature = 0.9 }# Engineering skills-llm_to_engineer = { model = "smart_llm", temperature = 0.2 }-llm_to_write = { model = "base-gemini", temperature = 0.2 }-llm_to_pipelex = { model = "base-claude", temperature = 0.2 }+llm_to_engineer = { model = "smart_llm_for_structured", temperature = 0.2 }llm_to_code = { model = "base-claude", temperature = 0.1 }llm_to_analyze_large_codebase = { model = "base-claude", temperature = 0.1 }-# Image skills-llm_for_basic_vision = { model = "gpt-4o-mini", temperature = 0.1 }-llm_to_write_img_gen_prompt = { model = "base-claude", temperature = 0.2 }-llm_to_describe_img = { model = "base-claude", temperature = 0.5 }-llm_for_visual_analysis = { model = "base-claude", temperature = 0.5 }-llm_for_visual_design = { model = "base-claude", temperature = 0.5 }-llm_to_design_fashion = { model = "base-claude", temperature = 0.7 }+# Vision skills+llm_for_img_to_text_cheap = { model = "gpt-4o-mini", temperature = 0.1 }llm_for_img_to_text = { model = "base-claude", temperature = 0.1 }+llm_for_diagram_to_text = { model = "base-claude", temperature = 0.3 }+llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }-# Extraction skills-llm_to_extract_diagram = { model = "base-claude", temperature = 0.5 }-llm_to_extract_invoice = { model = "claude-4.5-sonnet", temperature = 0.1 }-llm_to_extract_invoice_from_scan = { model = "base-claude", temperature = 0.5 }-llm_to_extract_legal_terms = { model = "base-claude", temperature = 0.1 }-llm_to_extract_tables = { model = "base-claude", temperature = 0.1 }+# Image generation prompting skills+llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }+llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }++# LLM Presets — For Testing ---------------------------------------------------------------------++llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }+llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.5 }+llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }+llm_for_testing_vision_structured = { model = "cheap_llm_for_vision", temperature = 0.5 }##################################################################################################### LLM Choices####################################################################################################[llm.choice_defaults]-for_text = "cheap_llm_for_text"-for_object = "cheap_llm_for_object"+for_text = "cheap_llm"+for_object = "cheap_llm_for_structured"##################################################################################################### Extract Presets@@ -144,7 +150,6 @@ choice_default = "gen_image_basic"[img_gen.presets]-# General purposegen_image_basic = { model = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }gen_image_fast = { model = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.tomlindex 4a77d257d..592533daf 100644--- a/.pipelex/inference/routing_profiles.toml+++ b/.pipelex/inference/routing_profiles.toml@@ -1,12 +1,12 @@# Routing profile library - Routes models to their backends-# ================================================+# =========================================================================================# This file controls which backend serves which model.# Simply change the 'active' field to switch profiles,# or you can add your own custom profiles.## Documentation: https://docs.pipelex.com# Support: https://go.pipelex.com/discord-# ================================================+# =========================================================================================# Which profile to use (change this to switch routing)active = "pipelex_first"@@ -21,13 +21,21 @@ active = "pipelex_first"# 3. The .pipelex/inference/backends.toml is already configured with api_key = "${PIPELEX_INFERENCE_API_KEY}"# which will get the key from the environment variable.-# ============================================+# =========================================================================================# Routing Profiles-# ============================================+# =========================================================================================[profiles.pipelex_first]description = "Use Pipelex Inference backend for all its supported models"default = "pipelex_inference"+fallback_order = [+ "pipelex_inference",+ "azure_openai",+ "bedrock",+ "blackboxai",+ "mistral",+ "fal",+][profiles.pipelex_first.routes]# Pattern matching: "model-pattern" = "backend-name"@@ -35,37 +43,82 @@ default = "pipelex_inference""claude-*" = "pipelex_inference""grok-*" = "pipelex_inference""gemini-*" = "pipelex_inference"++[profiles.pipelex_first.optional_routes] # Each optional route is considered only if its backend is available"*-sdxl" = "blackboxai""flux-*" = "blackboxai""nano-banana" = "blackboxai""gpt-image-1" = "openai""mistral-ocr" = "mistral"-[profiles.all_blackboxai]-description = "Use BlackBoxAI backend for all its supported models"-default = "blackboxai"+[profiles.all_anthropic]+description = "Use Anthropic backend for all its supported models"+default = "anthropic"++[profiles.all_azure_openai]+description = "Use Azure OpenAI backend for all its supported models"+default = "azure_openai"[profiles.all_bedrock]description = "Use Bedrock backend for all its supported models"default = "bedrock"-[profiles.all_anthropic]-description = "Use Anthropic backend for all its supported models"-default = "anthropic"+[profiles.all_blackboxai]+description = "Use BlackBoxAI backend for all its supported models"+default = "blackboxai"++[profiles.all_fal]+description = "Use FAL backend for all its supported models"+default = "fal"-[profiles.all_gemini]+[profiles.all_google]description = "Use Google GenAI backend for all its supported models"default = "google"-[profiles.all_azure_openai]-description = "Use Azure OpenAI backend for all its supported models"-default = "azure_openai"+[profiles.all_mistral]+description = "Use Mistral backend for all its supported models"+default = "mistral"++[profiles.all_ollama]+description = "Use Ollama backend for all its supported models"+default = "ollama"++[profiles.all_openai]+description = "Use OpenAI backend for all its supported models"+default = "openai"++[profiles.all_vertexai]+description = "Use Vertex AI backend for all its supported models"+default = "vertexai"++[profiles.all_xai]+description = "Use xAI backend for all its supported models"+default = "xai"++# =========================================================================================+# Custom Profiles+# =========================================================================================+# Add your own profiles below following the same pattern:+#+# [profiles.your_profile_name]+# description = "What this profile does"+# default = "backend-name" # Where to route models by default+# [profiles.your_profile_name.routes]+# "model-pattern" = "backend-name" # Specific routing rules+#+# Pattern matching supports:+# - Exact names: "gpt-4o-mini"+# - Wildcards: "claude-*" (matches all models starting with claude-)+# - Partial wildcards: "*-sonnet" (matches all sonnet variants)-[profiles.custom_routing]-description = "Custom routing"+# =========================================================================================+# Example of a custom routing profile with mostly pattern matching and one specific model+# =========================================================================================+[profiles.example_routing_using_patterns]+description = "Example routing profile using patterns"default = "pipelex_inference"-[profiles.custom_routing.routes]+[profiles.example_routing_using_patterns.routes]# Pattern matching: "model-pattern" = "backend-name""gpt-*" = "azure_openai""claude-*" = "bedrock"@@ -75,10 +128,14 @@ default = "pipelex_inference""flux-*" = "fal""gpt-image-1" = "openai"-[profiles.to_test_various_backends]-description = "Route models to various backends"+# =========================================================================================+# Example of a custom routing profile with specific model matching+# =========================================================================================++[profiles.example_routing_using_specific_models]+description = "Example routing profile using specific models"-[profiles.to_test_various_backends.routes]+[profiles.example_routing_using_specific_models.routes]"gpt-5-nano" = "pipelex_inference""gpt-4o-mini" = "blackboxai""gpt-5-mini" = "openai"@@ -93,19 +150,3 @@ description = "Route models to various backends""grok-3" = "pipelex_inference""grok-3-mini" = "xai"--# ============================================-# Custom Profiles-# ============================================-# Add your own profiles below following the same pattern:-#-# [profiles.your_profile_name]-# description = "What this profile does"-# default = "backend-name" # Where to route models by default-# [profiles.your_profile_name.routes]-# "model-pattern" = "backend-name" # Specific routing rules-#-# Pattern matching supports:-# - Exact names: "gpt-4o-mini"-# - Wildcards: "claude-*" (matches all models starting with claude-)-# - Partial wildcards: "*-sonnet" (matches all sonnet variants)diff --git a/.vscode/launch.json b/.vscode/launch.jsonindex 1827cff67..def33534c 100644--- a/.vscode/launch.json+++ b/.vscode/launch.json@@ -47,9 +47,7 @@"program": "${workspaceFolder}/.venv/bin/pipelex","args": ["validate",- "describe_image",- "-c",- "pipelex/libraries",+ "temp/bundle.plx",],"console": "integratedTerminal","justMyCode": falsediff --git a/.windsurfrules.md b/.windsurfrules.mdindex 23c6f20aa..6e236d028 100644--- a/.windsurfrules.md+++ b/.windsurfrules.md@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/AGENTS.md b/AGENTS.mdindex 23c6f20aa..6e236d028 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4a6688e78..e57c5b0e3 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,10 @@# Changelog+## Unreleased++- Updated the api documentation.+- Updated the routing profiles.+## [v0.14.3] - 2025-10-29### Addeddiff --git a/CLAUDE.md b/CLAUDE.mdindex 23c6f20aa..6e236d028 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -818,7 +818,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```@@ -1145,7 +1145,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/Makefile b/Makefileindex 348132ee4..6b104d414 100644--- a/Makefile+++ b/Makefile@@ -458,7 +458,7 @@ check-TODOs: envdocs: env$(call PRINT_TITLE,"Serving documentation with mkdocs")- $(VENV_MKDOCS) serve -a 127.0.0.1:8000 -f $(CURDIR)/mkdocs.yml --watch $(CURDIR)/docs -s+ $(VENV_MKDOCS) serve -a 127.0.0.1:8000 -f "$(CURDIR)/mkdocs.yml" --watch "$(CURDIR)/docs" -sdocs-check: env$(call PRINT_TITLE,"Checking documentation build with mkdocs")diff --git a/README.md b/README.mdindex 074eaaabc..7dce15f0d 100644--- a/README.md+++ b/README.md@@ -137,7 +137,7 @@ Performs comprehensive analysis comparing the candidate's CV against the job off"""inputs = { cv_pages = "Page[]", job_offer_pages = "Page[]" }output = "MatchAnalysis"-model = "llm_to_answer_hard_questions"+model = "llm_to_answer_questions"system_prompt = """You are an expert HR analyst and recruiter specializing in candidate-job fit assessment. Your task is to generate a structured MatchAnalysis comparing a candidate's CV against job requirements."""@@ -160,7 +160,7 @@ Generates exactly 5 targeted, relevant interview questions based on the match an"""inputs = { match_analysis = "MatchAnalysis" }output = "Question[5]"-model = "llm_to_write_questions"+model = "llm_for_writing_cheap"system_prompt = """You are an expert HR interviewer and talent assessment specialist. Your task is to generate structured interview questions based on candidate-position match analysis."""diff --git a/pipelex/builder/builder_errors.py b/pipelex/builder/builder_errors.pyindex 86c1b8b64..79457e77d 100644--- a/pipelex/builder/builder_errors.py+++ b/pipelex/builder/builder_errors.py@@ -20,7 +20,7 @@ from pipelex.builder.validation_error_data import (StaticValidationErrorData,)from pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.validation_errors import ValidationErrorDetailsProtocol+from pipelex.core.validation_errors import PipelexValidationExceptionAbstractfrom pipelex.exceptions import PipelexExceptionfrom pipelex.types import Self@@ -72,7 +72,7 @@ class ValidateDryRunError(Exception):"""Raised when validating the dry run of a pipe."""-class PipelexBundleError(PipelexException, ValidationErrorDetailsProtocol):+class PipelexBundleError(PipelexException, PipelexValidationExceptionAbstract):"""Main bundle error that aggregates multiple types of errors."""def __init__(diff --git a/pipelex/builder/pipe/pipe_llm_spec.py b/pipelex/builder/pipe/pipe_llm_spec.pyindex 03bb5d864..4d2fce330 100644--- a/pipelex/builder/pipe/pipe_llm_spec.py+++ b/pipelex/builder/pipe/pipe_llm_spec.py@@ -14,15 +14,14 @@ if TYPE_CHECKING:class LLMSkill(StrEnum):LLM_TO_RETRIEVE = "llm_to_retrieve"- LLM_TO_ANSWER_EASY_QUESTIONS = "llm_to_answer_easy_questions"- LLM_TO_ANSWER_HARD_QUESTIONS = "llm_to_answer_hard_questions"- LLM_TO_WRITE_QUESTIONS = "llm_to_write_questions"- LLM_FOR_BASIC_VISION = "llm_for_basic_vision"- LLM_FOR_VISUAL_ANALYSIS = "llm_for_visual_analysis"++ LLM_TO_ANSWER_QUESTIONS_CHEAP = "llm_to_answer_questions_cheap"+ LLM_TO_ANSWER_QUESTIONS = "llm_to_answer_questions"++ LLM_FOR_WRITING_CHEAP = "llm_for_writing_cheap"+ LLM_FOR_IMG_TO_TEXT_CHEAP = "llm_for_img_to_text_cheap"LLM_FOR_VISUAL_DESIGN = "llm_for_visual_design"LLM_FOR_CREATIVE_WRITING = "llm_for_creative_writing"- LLM_TO_REASON_ON_DIAGRAM = "llm_to_reason_on_diagram"- LLM_TO_ANALYZE_DATA = "llm_to_analyze_data"LLM_TO_CODE = "llm_to_code"LLM_TO_ANALYZE_LARGE_CODEBASE = "llm_to_analyze_large_codebase"diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 6ce815e04..ed131e5c6 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -8,7 +8,8 @@ from typing_extensions import overridefrom pipelex.cli.commands.build_cmd import build_appfrom pipelex.cli.commands.doctor_cmd import doctor_cmd-from pipelex.cli.commands.init_cmd import InitFocus, init_cmd+from pipelex.cli.commands.init.command import init_cmd+from pipelex.cli.commands.init.ui.types import InitFocusfrom pipelex.cli.commands.kit_cmd import kit_appfrom pipelex.cli.commands.run_cmd import run_cmdfrom pipelex.cli.commands.show_cmd import show_appdiff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex 09a1bd842..7c66ac744 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -4,15 +4,29 @@ from typing import TYPE_CHECKING, Annotatedimport clickimport typer-from posthog import new_context, tag+from posthog import tagfrom pipelex import pretty_printfrom pipelex.builder.builder import PipelexBundleSpec, load_and_validate_bundlefrom pipelex.builder.builder_errors import PipeBuilderError, PipelexBundleErrorfrom pipelex.builder.builder_loop import BuilderLoopfrom pipelex.builder.runner_code import generate_input_memory_json_string, generate_runner_code-from pipelex.exceptions import PipeInputError, PipelineExecutionError-from pipelex.hub import get_report_delegate, get_required_pipe+from pipelex.cli.error_handlers import (+ ErrorContext,+ handle_model_availability_error,+ handle_model_choice_error,+ handle_model_deck_preset_error,+ handle_validation_error,+)+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError+from pipelex.exceptions import (+ LibraryLoadingError,+ PipeInputError,+ PipelineExecutionError,+ PipeOperatorModelAvailabilityError,+ PipeOperatorModelChoiceError,+)+from pipelex.hub import get_report_delegate, get_required_pipe, get_telemetry_managerfrom pipelex.language.plx_factory import PlxFactoryfrom pipelex.pipelex import PACKAGE_VERSION, Pipelexfrom pipelex.pipeline.execute import execute_pipeline@@ -82,7 +96,13 @@ def build_pipe_cmd(typer.Option("--no-output", help="Skip saving the pipeline to file"),] = False,) -> None:- Pipelex.make(integration_mode=IntegrationMode.CLI)+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PIPE)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PIPE)+typer.secho("🔥 Starting pipe builder... 🚀\n", fg=typer.colors.GREEN)async def run_pipeline():@@ -136,17 +156,27 @@ def build_pipe_cmd(except Exception as exc:typer.secho(f"⚠️ Warning: Could not generate input JSON: {exc}", fg=typer.colors.YELLOW)- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPE}")+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPE}")++ start_time = time.time()+ asyncio.run(run_pipeline())+ end_time = time.time()+ typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)++ get_report_delegate().generate_report()++ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.BUILD)- start_time = time.time()- asyncio.run(run_pipeline())- end_time = time.time()- typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)+ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.BUILD)- get_report_delegate().generate_report()+ finally:+ pipelex_instance.teardown()@build_app.command(SUB_COMMAND_RUNNER, help="Build the Python code to run a pipe with the necessary inputs")@@ -229,9 +259,6 @@ def prepare_runner_cmd(raise typer.Exit(1)async def prepare_runner(pipe_code: str | None = None, bundle_path: str | None = None):- # Initialize Pipelex- Pipelex.make(integration_mode=IntegrationMode.CLI)-if bundle_path:try:bundle_blueprint = await load_and_validate_bundle(bundle_path)@@ -287,12 +314,29 @@ def prepare_runner_cmd(typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)raise typer.Exit(1) from exc- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)++ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")++ asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))- asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))+ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.BUILD)++ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.BUILD)++ finally:+ pipelex_instance.teardown()@build_app.command(SUB_COMMAND_INPUTS, help="Generate example input JSON for a pipe")@@ -371,9 +415,6 @@ def generate_inputs_cmd(raise typer.Exit(1)async def generate_inputs(pipe_code: str | None = None, bundle_path: str | None = None):- # Initialize Pipelex- Pipelex.make(integration_mode=IntegrationMode.CLI)-if bundle_path:try:bundle_blueprint = await load_and_validate_bundle(bundle_path)@@ -425,12 +466,29 @@ def generate_inputs_cmd(typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)raise typer.Exit(1) from exc- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)- asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")++ asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))++ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.BUILD)++ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.BUILD)++ finally:+ pipelex_instance.teardown()@build_app.command(SUB_COMMAND_ONE_SHOT_PIPE, help="Developer utility for contributors: deliver pipeline in one shot, without validation loop")@@ -452,7 +510,13 @@ def build_one_shot_cmd(typer.Option("--no-output", help="Skip saving the pipeline to file"),] = False,) -> None:- Pipelex.make(integration_mode=IntegrationMode.CLI)+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_ONE_SHOT)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_ONE_SHOT)+typer.secho("🔥 Starting pipe builder... 🚀\n", fg=typer.colors.GREEN)async def run_pipeline():@@ -484,17 +548,27 @@ def build_one_shot_cmd(save_text_to_path(text=plx_content, path=output_path)typer.secho(f"\n✅ Pipeline saved to: {output_path}", fg=typer.colors.GREEN)- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_ONE_SHOT_PIPE}")+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_ONE_SHOT_PIPE}")++ start_time = time.time()+ asyncio.run(run_pipeline())+ end_time = time.time()+ typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)- start_time = time.time()- asyncio.run(run_pipeline())- end_time = time.time()- typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)+ get_report_delegate().generate_report()- get_report_delegate().generate_report()+ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.BUILD)++ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.BUILD)++ finally:+ pipelex_instance.teardown()@build_app.command(@@ -527,7 +601,13 @@ def build_partial_cmd(typer.Option("--no-output", help="Skip saving the pipeline to file"),] = False,) -> None:- Pipelex.make(integration_mode=IntegrationMode.CLI)+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PARTIAL)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_PARTIAL)+typer.secho("🔥 Starting pipe builder... 🚀\n", fg=typer.colors.GREEN)async def run_pipeline():@@ -577,14 +657,24 @@ def build_partial_cmd(else:typer.secho("\n⚠️ Pipeline not saved to file (--no-output specified)", fg=typer.colors.YELLOW)- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PARTIAL_PIPE}")+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PARTIAL_PIPE}")++ start_time = time.time()+ asyncio.run(run_pipeline())+ end_time = time.time()+ typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)++ get_report_delegate().generate_report()++ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.BUILD)- start_time = time.time()- asyncio.run(run_pipeline())- end_time = time.time()- typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN)+ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.BUILD)- get_report_delegate().generate_report()+ finally:+ pipelex_instance.teardown()diff --git a/pipelex/cli/commands/doctor_cmd.py b/pipelex/cli/commands/doctor_cmd.pyindex d961cd437..cc6daab43 100644--- a/pipelex/cli/commands/doctor_cmd.py+++ b/pipelex/cli/commands/doctor_cmd.py@@ -12,10 +12,16 @@ from rich.panel import Panelfrom rich.prompt import Confirmfrom rich.text import Text-from pipelex.cli.commands.init_cmd import InitFocus, init_cmd, init_config+from pipelex import log+from pipelex.cli.commands.init.command import init_cmd+from pipelex.cli.commands.init.config_files import init_config+from pipelex.cli.commands.init.ui.types import InitFocusfrom pipelex.cogt.model_backends.backend_library import BackendCredentialsReport-from pipelex.config import PipelexConfig+from pipelex.cogt.models.model_manager import ModelManager+from pipelex.config import PipelexConfig, get_configfrom pipelex.core.validation import report_validation_error+from pipelex.exceptions import PipelexConfigError+from pipelex.hub import PipelexHub, set_pipelex_hubfrom pipelex.system.configuration.config_loader import config_managerfrom pipelex.system.environment import get_optional_envfrom pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryConfig@@ -23,6 +29,7 @@ from pipelex.tools.misc.dict_utils import extract_vars_from_strings_recursivefrom pipelex.tools.misc.file_utils import path_existsfrom pipelex.tools.misc.placeholder import value_is_placeholderfrom pipelex.tools.misc.toml_utils import load_toml_from_path+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProviderfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error@@ -166,6 +173,8 @@ def display_health_report(backends_healthy: bool,backends_message: str,backend_reports: dict[str, BackendCredentialsReport],+ models_healthy: bool,+ models_message: str,) -> None:"""Display a comprehensive health report.@@ -179,8 +188,10 @@ def display_health_report(backends_healthy: Whether backends check passedbackends_message: Message about backends statusbackend_reports: Dict of backend credential reports+ models_healthy: Whether models check passed+ models_message: Message about models status"""- all_healthy = config_healthy and telemetry_healthy and backends_healthy+ all_healthy = config_healthy and telemetry_healthy and backends_healthy and models_healthy# Overall status panelif all_healthy:@@ -238,6 +249,14 @@ def display_health_report(console.print(f" [yellow]⚠[/yellow] Placeholders: {', '.join(backend_report.placeholder_vars)}")console.print()+ # Models section+ console.print("[bold]Models[/bold]")+ if models_healthy:+ console.print(f" [green]✓[/green] {models_message}")+ else:+ console.print(f" [red]✗[/red] {models_message}")+ console.print()+# Recommended actionsif not all_healthy:# Check what can be auto-fixed@@ -300,6 +319,35 @@ def display_health_report(console.print()+def check_models() -> tuple[bool, str]:+ """Check if models are valid.++ Returns:+ Tuple of (is_healthy, message)+ """+ pipelex_hub = PipelexHub()+ set_pipelex_hub(pipelex_hub)++ # tools+ try:+ pipelex_hub.setup_config(config_cls=PipelexConfig)+ except ValidationError as validation_error:+ validation_error_msg = report_validation_error(category="config", validation_error=validation_error)+ msg = f"Could not setup config because of: {validation_error_msg}"+ raise PipelexConfigError(msg) from validation_error++ log.configure(log_config=get_config().pipelex.log_config)++ models_manager = ModelManager()+ secrets_provider = EnvSecretsProvider()+ try:+ models_manager.setup(secrets_provider=secrets_provider)+ models_manager.validate_model_deck()+ except Exception as exc:+ return False, f"Error checking models: {exc}"+ return True, "Models are valid"++def doctor_cmd(fix: bool = False,) -> None:@@ -309,142 +357,158 @@ def doctor_cmd(fix: If True, offer to fix detected issues interactively"""console = Console()-+ do_doctor_cmd(console=console, fix=fix)try:- # Run health checks- config_healthy, config_missing_count, config_message = check_config_files()- telemetry_healthy, telemetry_message = check_telemetry_config()- backends_healthy, backend_reports, backends_message = check_backend_credentials()-- # Display report- display_health_report(- console=console,- config_healthy=config_healthy,- config_message=config_message,- config_missing_count=config_missing_count,- telemetry_healthy=telemetry_healthy,- telemetry_message=telemetry_message,- backends_healthy=backends_healthy,- backends_message=backends_message,- backend_reports=backend_reports,- )+ do_doctor_cmd(console=console, fix=fix)- all_healthy = config_healthy and telemetry_healthy and backends_healthy+ except Exception as exc:+ # Handle unexpected errors gracefully without printing traces+ console.print()+ console.print(f"[red]✗ Unexpected error: {exc!s}[/red]")+ console.print()+ console.print("[dim]If you need help:[/dim]")+ console.print(" [cyan]https://docs.pipelex.com[/cyan] - Documentation")+ console.print(" [cyan]https://go.pipelex.com/discord[/cyan] - Discord Community")+ console.print()+ sys.exit(1)- # Exit code: 0 if healthy, 1 if issues found- if all_healthy:- sys.exit(0)- # Determine what can be auto-fixed- can_fix_config = not config_healthy and config_missing_count > 0- can_fix_telemetry = not telemetry_healthy and "not found" in telemetry_message.lower()- has_auto_fixable_issues = can_fix_config or can_fix_telemetry+def do_doctor_cmd(+ console: Console,+ fix: bool = False,+) -> None:+ """Check Pipelex configuration health and suggest fixes.- # Determine what requires manual fixes- has_config_validation_error = not config_healthy and config_missing_count == 0- has_telemetry_validation_error = not telemetry_healthy and "not found" not in telemetry_message.lower()- has_backend_credential_issues = not backends_healthy and backend_reports+ Args:+ console: Rich Console instance for output+ fix: If True, offer to fix detected issues interactively+ """+ # Run health checks+ config_healthy, config_missing_count, config_message = check_config_files()+ telemetry_healthy, telemetry_message = check_telemetry_config()+ backends_healthy, backend_reports, backends_message = check_backend_credentials()+ models_healthy, models_message = check_models()++ # Display report+ display_health_report(+ console=console,+ config_healthy=config_healthy,+ config_message=config_message,+ config_missing_count=config_missing_count,+ telemetry_healthy=telemetry_healthy,+ telemetry_message=telemetry_message,+ backends_healthy=backends_healthy,+ backends_message=backends_message,+ backend_reports=backend_reports,+ models_healthy=models_healthy,+ models_message=models_message,+ )- # If --fix flag is provided, offer to fix auto-fixable issues- if fix and has_auto_fixable_issues:- console.print("[bold yellow]Interactive Fix Mode[/bold yellow]")- console.print()+ all_healthy = config_healthy and telemetry_healthy and backends_healthy and models_healthy- # Fix missing config files- if can_fix_config:- if Confirm.ask(f"[bold]Install {config_missing_count} missing configuration file(s)?[/bold]", default=True):- try:- console.print()- init_cmd(focus=InitFocus.CONFIG, reset=False, skip_confirmation=True)- console.print("[green]✓[/green] Configuration files installed")- except Exception as exc:- console.print(f"[red]Failed to install configuration files: {exc!s}[/red]")- console.print()+ # Exit code: 0 if healthy, 1 if issues found+ if all_healthy:+ sys.exit(0)- # Fix missing telemetry config- if can_fix_telemetry:- if Confirm.ask("[bold]Configure telemetry preferences?[/bold]", default=True):- try:- console.print()- init_cmd(focus=InitFocus.TELEMETRY, reset=False, skip_confirmation=True)- console.print("[green]✓[/green] Telemetry configured")- except Exception as exc:- console.print(f"[red]Failed to configure telemetry: {exc!s}[/red]")- console.print()+ # Determine what can be auto-fixed+ can_fix_config = not config_healthy and config_missing_count > 0+ can_fix_telemetry = not telemetry_healthy and "not found" in telemetry_message.lower()+ has_auto_fixable_issues = can_fix_config or can_fix_telemetry- # Handle issues that can't be auto-fixed- if has_config_validation_error or has_telemetry_validation_error or has_backend_credential_issues:- console.print("[bold yellow]Manual Fixes Required[/bold yellow]")- console.print()+ # Determine what requires manual fixes+ has_config_validation_error = not config_healthy and config_missing_count == 0+ has_telemetry_validation_error = not telemetry_healthy and "not found" not in telemetry_message.lower()+ has_backend_credential_issues = not backends_healthy and backend_reports- # Config validation errors- if has_config_validation_error:- console.print("[bold]Configuration validation error:[/bold]")- console.print(f" {config_message}")- console.print()- console.print("You can fix this manually by editing [cyan].pipelex/pipelex.toml[/cyan]")- console.print("or run [cyan]pipelex init config --reset[/cyan] to regenerate from template.")- console.print()+ # If --fix flag is provided, offer to fix auto-fixable issues+ if fix and has_auto_fixable_issues:+ console.print("[bold yellow]Interactive Fix Mode[/bold yellow]")+ console.print()- # Telemetry validation errors- if has_telemetry_validation_error:- console.print("[bold]Telemetry validation error:[/bold]")- console.print(f" {telemetry_message}")+ # Fix missing config files+ if can_fix_config:+ if Confirm.ask(f"[bold]Install {config_missing_count} missing configuration file(s)?[/bold]", default=True):+ try:+ console.print()+ init_cmd(focus=InitFocus.CONFIG, reset=False, skip_confirmation=True)+ console.print("[green]✓[/green] Configuration files installed")+ except Exception as exc:+ console.print(f"[red]Failed to install configuration files: {exc!s}[/red]")console.print()- console.print("You can fix this manually by editing [cyan].pipelex/telemetry.toml[/cyan]")- console.print("or run [cyan]pipelex init telemetry --reset[/cyan] to regenerate from template.")++ # Fix missing telemetry config+ if can_fix_telemetry:+ if Confirm.ask("[bold]Configure telemetry preferences?[/bold]", default=True):+ try:+ console.print()+ init_cmd(focus=InitFocus.TELEMETRY, reset=False, skip_confirmation=True)+ console.print("[green]✓[/green] Telemetry configured")+ except Exception as exc:+ console.print(f"[red]Failed to configure telemetry: {exc!s}[/red]")console.print()- # Backend credentials- if has_backend_credential_issues:- all_missing_vars: set[str] = set()- for backend_report in backend_reports.values():- if not backend_report.all_credentials_valid:- all_missing_vars.update(backend_report.missing_vars)+ # Handle issues that can't be auto-fixed+ if has_config_validation_error or has_telemetry_validation_error or has_backend_credential_issues:+ console.print("[bold yellow]Manual Fixes Required[/bold yellow]")+ console.print()- if all_missing_vars:- console.print("[bold]Backend credentials:[/bold]")- console.print()- console.print("Set the following environment variables:")- console.print()+ # Config validation errors+ if has_config_validation_error:+ console.print("[bold]Configuration validation error:[/bold]")+ console.print(f" {config_message}")+ console.print()+ console.print("You can fix this manually by editing [cyan].pipelex/pipelex.toml[/cyan]")+ console.print("or run [cyan]pipelex init config --reset[/cyan] to regenerate from template.")+ console.print()- # Show .env file syntax first- console.print("[dim]# In your .env file:[/dim]")- for var_name in sorted(all_missing_vars):- console.print(f"{var_name}=[yellow]your_value_here[/yellow]")- console.print()+ # Telemetry validation errors+ if has_telemetry_validation_error:+ console.print("[bold]Telemetry validation error:[/bold]")+ console.print(f" {telemetry_message}")+ console.print()+ console.print("You can fix this manually by editing [cyan].pipelex/telemetry.toml[/cyan]")+ console.print("or run [cyan]pipelex init telemetry --reset[/cyan] to regenerate from template.")+ console.print()- # Show shell syntax for different platforms- console.print("[dim]# Or in your shell:[/dim]")- console.print()+ # Backend credentials+ if has_backend_credential_issues:+ all_missing_vars: set[str] = set()+ for backend_report in backend_reports.values():+ if not backend_report.all_credentials_valid:+ all_missing_vars.update(backend_report.missing_vars)- # Linux/MacOS- console.print("[dim]# Linux/MacOS[/dim]")- for var_name in sorted(all_missing_vars):- console.print(f"export {var_name}=[yellow]your_value_here[/yellow]")- console.print()+ if all_missing_vars:+ console.print("[bold]Backend credentials:[/bold]")+ console.print()+ console.print("Set the following environment variables:")+ console.print()- # Windows PowerShell- console.print("[dim]# Windows PowerShell[/dim]")- for var_name in sorted(all_missing_vars):- console.print(f'$env:{var_name}="[yellow]your_value_here[/yellow]"')- console.print()+ # Show .env file syntax first+ console.print("[dim]# In your .env file:[/dim]")+ for var_name in sorted(all_missing_vars):+ console.print(f"{var_name}=[yellow]your_value_here[/yellow]")+ console.print()- # Windows CMD- console.print("[dim]# Windows CMD[/dim]")- for var_name in sorted(all_missing_vars):- console.print(f"set {var_name}=[yellow]your_value_here[/yellow]")- console.print()+ # Show shell syntax for different platforms+ console.print("[dim]# Or in your shell:[/dim]")+ console.print()- sys.exit(1)+ # Linux/MacOS+ console.print("[dim]# Linux/MacOS[/dim]")+ for var_name in sorted(all_missing_vars):+ console.print(f"export {var_name}=[yellow]your_value_here[/yellow]")+ console.print()- except Exception as exc:- # Handle unexpected errors gracefully without printing traces- console.print()- console.print(f"[red]✗ Unexpected error: {exc!s}[/red]")- console.print()- console.print("[dim]If you need help:[/dim]")- console.print(" [cyan]https://docs.pipelex.com[/cyan] - Documentation")- console.print(" [cyan]https://go.pipelex.com/discord[/cyan] - Discord Community")- console.print()- sys.exit(1)+ # Windows PowerShell+ console.print("[dim]# Windows PowerShell[/dim]")+ for var_name in sorted(all_missing_vars):+ console.print(f'$env:{var_name}="[yellow]your_value_here[/yellow]"')+ console.print()++ # Windows CMD+ console.print("[dim]# Windows CMD[/dim]")+ for var_name in sorted(all_missing_vars):+ console.print(f"set {var_name}=[yellow]your_value_here[/yellow]")+ console.print()++ sys.exit(1)diff --git a/pipelex/cli/commands/init/__init__.py b/pipelex/cli/commands/init/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/cli/commands/init/backends.py b/pipelex/cli/commands/init/backends.pynew file mode 100644index 000000000..b07849f43--- /dev/null+++ b/pipelex/cli/commands/init/backends.py@@ -0,0 +1,100 @@+"""Backend configuration logic for the init command."""++import os+from typing import Any++from rich.console import Console++from pipelex.cli.commands.init.ui.backends_ui import (+ build_backend_selection_panel,+ display_selected_backends,+ get_backend_options_from_toml,+ get_currently_enabled_backends,+ prompt_backend_indices,+)+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.system.configuration.config_loader import config_manager+from pipelex.tools.misc.file_utils import path_exists+from pipelex.tools.misc.toml_utils import load_toml_from_path, load_toml_with_tomlkit, save_toml_to_path+++def update_backends_in_toml(toml_doc: Any, selected_indices: list[int], backend_options: list[tuple[str, str]]) -> None:+ """Update the backends.toml document with selected backends.++ Args:+ toml_doc: The TOML document to update.+ selected_indices: List of backend indices to enable.+ backend_options: List of available backend options.+ """+ selected_backend_keys = {backend_options[idx][0] for idx in selected_indices}++ # Disable all backends first (except internal)+ for backend_key in toml_doc:+ if backend_key != "internal" and backend_key in toml_doc:+ backend_section = toml_doc[backend_key]+ # Set enabled field based on selection (works with tomlkit's special types)+ backend_section["enabled"] = backend_key in selected_backend_keys # type: ignore[index]+++def get_selected_backend_keys(backends_toml_path: str) -> list[str]:+ """Extract the list of enabled backend keys from backends.toml.++ Args:+ backends_toml_path: Path to the backends.toml file.++ Returns:+ List of backend keys that are enabled (excluding 'internal').+ """+ selected_backends: list[str] = []++ if not path_exists(backends_toml_path):+ return selected_backends++ toml_doc = load_toml_from_path(backends_toml_path)++ for backend_key in toml_doc:+ if backend_key != "internal":+ backend_section = toml_doc[backend_key]+ if isinstance(backend_section, dict):+ if backend_section.get("enabled", False) is True: # pyright: ignore[reportUnknownMemberType]+ selected_backends.append(backend_key)++ return selected_backends+++def customize_backends_config() -> None:+ """Interactively customize which inference backends are enabled in backends.toml."""+ console = Console()+ backends_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "backends.toml")+ template_backends_path = os.path.join(str(get_kit_configs_dir()), "inference", "backends.toml")++ if not path_exists(backends_toml_path):+ console.print("[yellow]⚠ Warning: backends.toml not found, skipping backend customization[/yellow]")+ return++ try:+ # Get backend options from template and existing config+ existing_path = backends_toml_path if path_exists(backends_toml_path) else None+ backend_options = get_backend_options_from_toml(template_backends_path, existing_path)++ # Get currently enabled backends to show user their current selection+ currently_enabled = get_currently_enabled_backends(backends_toml_path, backend_options)++ # Load the backends.toml file+ toml_doc = load_toml_with_tomlkit(backends_toml_path)+ console.print()++ # UI: Display panel and get user selection+ console.print(build_backend_selection_panel(backend_options, currently_enabled))+ selected_indices = prompt_backend_indices(console, backend_options, currently_enabled)++ # Business logic: Update TOML+ update_backends_in_toml(toml_doc, selected_indices, backend_options)+ save_toml_to_path(toml_doc, backends_toml_path)++ # UI: Display confirmation+ display_selected_backends(console, selected_indices, backend_options)++ except Exception as exc:+ console.print(f"[yellow]⚠ Warning: Failed to customize backends: {exc}[/yellow]")+ console.print("[dim]You can manually edit .pipelex/inference/backends.toml later[/dim]")diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init/command.pysimilarity index 52%rename from pipelex/cli/commands/init_cmd.pyrename to pipelex/cli/commands/init/command.pyindex 7a7e198dc..093fddc99 100644--- a/pipelex/cli/commands/init_cmd.py+++ b/pipelex/cli/commands/init/command.py@@ -1,217 +1,62 @@+"""Main command orchestration for the init command."""+import os-import shutil-from typing import Anyimport typerfrom rich.console import Consolefrom rich.prompt import Confirm-from pipelex.cli.commands.init_ui import (- InitFocus,- build_backend_selection_panel,- build_initialization_panel,- build_telemetry_selection_panel,- display_already_configured_message,- display_selected_backends,- get_backend_options_from_toml,- get_currently_enabled_backends,- prompt_backend_indices,- prompt_telemetry_mode,-)-from pipelex.exceptions import PipelexCLIError-from pipelex.kit.paths import get_configs_dir+from pipelex.cli.commands.init.backends import customize_backends_config, get_selected_backend_keys+from pipelex.cli.commands.init.config_files import init_config+from pipelex.cli.commands.init.routing import customize_routing_profile+from pipelex.cli.commands.init.telemetry import setup_telemetry+from pipelex.cli.commands.init.ui.general_ui import build_initialization_panel, display_already_configured_message+from pipelex.cli.commands.init.ui.types import InitFocusfrom pipelex.system.configuration.config_loader import config_manager-from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryMode+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAMEfrom pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstractfrom pipelex.tools.misc.file_utils import path_exists-from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path---def update_backends_in_toml(toml_doc: Any, selected_indices: list[int], backend_options: list[tuple[str, str]]) -> None:- """Update the backends.toml document with selected backends.-- Args:- toml_doc: The TOML document to update.- selected_indices: List of backend indices to enable.- backend_options: List of available backend options.- """- selected_backend_keys = {backend_options[idx][0] for idx in selected_indices}-- # Disable all backends first (except internal)- for backend_key in toml_doc:- if backend_key != "internal" and backend_key in toml_doc:- backend_section = toml_doc[backend_key]- # Set enabled field based on selection (works with tomlkit's special types)- backend_section["enabled"] = backend_key in selected_backend_keys # type: ignore[index]---def customize_backends_config() -> None:- """Interactively customize which inference backends are enabled in backends.toml."""- console = Console()- backends_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "backends.toml")- template_backends_path = os.path.join(str(get_configs_dir()), "inference", "backends.toml")-- if not path_exists(backends_toml_path):- console.print("[yellow]⚠ Warning: backends.toml not found, skipping backend customization[/yellow]")- return-- try:- # Get backend options from template and existing config- existing_path = backends_toml_path if path_exists(backends_toml_path) else None- backend_options = get_backend_options_from_toml(template_backends_path, existing_path)-- # Get currently enabled backends to show user their current selection- currently_enabled = get_currently_enabled_backends(backends_toml_path, backend_options)-- # Load the backends.toml file- toml_doc = load_toml_with_tomlkit(backends_toml_path)- console.print()-- # UI: Display panel and get user selection- console.print(build_backend_selection_panel(backend_options, currently_enabled))- selected_indices = prompt_backend_indices(console, backend_options, currently_enabled)-- # Business logic: Update TOML- update_backends_in_toml(toml_doc, selected_indices, backend_options)- save_toml_to_path(toml_doc, backends_toml_path)-- # UI: Display confirmation- display_selected_backends(console, selected_indices, backend_options)-- except Exception as exc:- console.print(f"[yellow]⚠ Warning: Failed to customize backends: {exc}[/yellow]")- console.print("[dim]You can manually edit .pipelex/inference/backends.toml later[/dim]")---def init_config(reset: bool = False, dry_run: bool = False) -> int:- """Initialize pipelex configuration in the .pipelex directory. Does not install telemetry, just the main config dans inference backends.-- Args:- reset: Whether to overwrite existing files.- dry_run: Whether to only print the files that would be copied, without actually copying them.-- Returns:- The number of files copied.- """- config_template_dir = str(get_configs_dir())- target_config_dir = config_manager.pipelex_config_dir-- os.makedirs(target_config_dir, exist_ok=True)-- try:- copied_files: list[str] = []- existing_files: list[str] = []-- def copy_directory_structure(src_dir: str, dst_dir: str, relative_path: str = "", dry_run: bool = False) -> None:- """Recursively copy directory structure, handling existing files."""- for item in os.listdir(src_dir):- src_item = os.path.join(src_dir, item)- dst_item = os.path.join(dst_dir, item)- relative_item = os.path.join(relative_path, item) if relative_path else item-- # Skip telemetry.toml - it will be created when user is prompted- if item == TELEMETRY_CONFIG_FILE_NAME:- continue-- if os.path.isdir(src_item):- if not dry_run:- os.makedirs(dst_item, exist_ok=True)- copy_directory_structure(src_item, dst_item, relative_item, dry_run)- elif os.path.exists(dst_item) and not reset:- existing_files.append(relative_item)- else:- if not dry_run:- shutil.copy2(src_item, dst_item)- copied_files.append(relative_item)-- copy_directory_structure(src_dir=config_template_dir, dst_dir=target_config_dir, dry_run=dry_run)-- if dry_run:- return len(copied_files)-- # Report results- if copied_files:- typer.echo(f"✅ Copied {len(copied_files)} files to {target_config_dir}:")- for file in sorted(copied_files):- typer.echo(f" • {file}")-- if existing_files:- typer.echo(f"ℹ️ Skipped {len(existing_files)} existing files (use --reset to overwrite):")- for file in sorted(existing_files):- typer.echo(f" • {file}")-- if not copied_files and not existing_files:- typer.echo(f"✅ Configuration directory {target_config_dir} is already up to date")-- except Exception as exc:- msg = f"Failed to initialize configuration: {exc}"- raise PipelexCLIError(msg) from exc-- return len(copied_files)---def setup_telemetry(console: Console, telemetry_config_path: str) -> TelemetryMode:- """Set up telemetry configuration interactively.-- Args:- console: Rich Console instance for user interaction.- telemetry_config_path: Path to save the telemetry configuration.-- Returns:- The selected TelemetryMode.-- Raises:- typer.Exit: If user chooses to quit.- """- console.print()- console.print(build_telemetry_selection_panel())-- telemetry_mode = prompt_telemetry_mode(console)-- # Save telemetry config- template_path = os.path.join(str(get_configs_dir()), TELEMETRY_CONFIG_FILE_NAME)- toml_doc = load_toml_with_tomlkit(template_path)- toml_doc["telemetry_mode"] = telemetry_mode- save_toml_to_path(toml_doc, telemetry_config_path)-- console.print(f"\n[green]✓[/green] Telemetry mode set to: [bold cyan]{telemetry_mode}[/bold cyan]")-- return telemetry_modedef determine_needs(reset: bool,check_config: bool,check_inference: bool,+ check_routing: bool,check_telemetry: bool,backends_toml_path: str,+ routing_profiles_toml_path: str,telemetry_config_path: str,-) -> tuple[bool, bool, bool]:+) -> tuple[bool, bool, bool, bool]:"""Determine what needs to be initialized based on current state.Args:reset: Whether this is a reset operation.check_config: Whether to check config files.check_inference: Whether to check inference setup.+ check_routing: Whether to check routing setup.check_telemetry: Whether to check telemetry setup.backends_toml_path: Path to backends.toml file.+ routing_profiles_toml_path: Path to routing_profiles.toml file.telemetry_config_path: Path to telemetry config file.Returns:- Tuple of (needs_config, needs_inference, needs_telemetry) booleans.+ Tuple of (needs_config, needs_inference, needs_routing, needs_telemetry) booleans."""nb_missing_config_files = init_config(reset=False, dry_run=True) if check_config else 0needs_config = check_config and (nb_missing_config_files > 0 or reset)needs_inference = check_inference and (not path_exists(backends_toml_path) or reset)+ needs_routing = check_routing and (not path_exists(routing_profiles_toml_path) or reset)needs_telemetry = check_telemetry and (not path_exists(telemetry_config_path) or reset)- return needs_config, needs_inference, needs_telemetry+ return needs_config, needs_inference, needs_routing, needs_telemetrydef handle_already_configured(focus: InitFocus,console: Console,backends_toml_path: str,+ routing_profiles_toml_path: str,telemetry_config_path: str,) -> bool:"""Handle the case when everything is already configured.@@ -220,6 +65,7 @@ def handle_already_configured(focus: The initialization focus area.console: Rich Console instance for output.backends_toml_path: Path to backends.toml file.+ routing_profiles_toml_path: Path to routing_profiles.toml file.telemetry_config_path: Path to telemetry config file.Returns:@@ -228,6 +74,7 @@ def handle_already_configured(# Map focus to config path for displayconfig_path_map = {InitFocus.INFERENCE: backends_toml_path,+ InitFocus.ROUTING: routing_profiles_toml_path,InitFocus.TELEMETRY: telemetry_config_path,InitFocus.CONFIG: ".pipelex/",}@@ -236,26 +83,28 @@ def handle_already_configured(return display_already_configured_message(focus, console, config_path)-def update_needs_for_reconfigure(focus: InitFocus) -> tuple[bool, bool, bool]:+def update_needs_for_reconfigure(focus: InitFocus) -> tuple[bool, bool, bool, bool]:"""Update needs flags when user wants to reconfigure.Args:focus: The initialization focus area.Returns:- Tuple of (needs_config, needs_inference, needs_telemetry) booleans.+ Tuple of (needs_config, needs_inference, needs_routing, needs_telemetry) booleans."""needs_config = focus == InitFocus.CONFIGneeds_inference = focus == InitFocus.INFERENCE+ needs_routing = focus == InitFocus.ROUTINGneeds_telemetry = focus == InitFocus.TELEMETRY- return needs_config, needs_inference, needs_telemetry+ return needs_config, needs_inference, needs_routing, needs_telemetrydef confirm_initialization(console: Console,needs_config: bool,needs_inference: bool,+ needs_routing: bool,needs_telemetry: bool,reset: bool,focus: InitFocus,@@ -266,6 +115,7 @@ def confirm_initialization(console: Rich Console instance for user interaction.needs_config: Whether config initialization is needed.needs_inference: Whether inference setup is needed.+ needs_routing: Whether routing setup is needed.needs_telemetry: Whether telemetry setup is needed.reset: Whether this is a reset operation.focus: The initialization focus area.@@ -277,15 +127,15 @@ def confirm_initialization(typer.Exit: If user cancels initialization."""console.print()- console.print(build_initialization_panel(needs_config, needs_inference, needs_telemetry, reset))+ console.print(build_initialization_panel(needs_config, needs_inference, needs_routing, needs_telemetry, reset))if not Confirm.ask("[bold]Continue with initialization?[/bold]", default=True):console.print("\n[yellow]Initialization cancelled.[/yellow]")- if needs_config or needs_inference or needs_telemetry:+ if needs_config or needs_inference or needs_routing or needs_telemetry:match focus:case InitFocus.ALL:init_cmd_str = "pipelex init"- case InitFocus.CONFIG | InitFocus.INFERENCE | InitFocus.TELEMETRY:+ case InitFocus.CONFIG | InitFocus.INFERENCE | InitFocus.ROUTING | InitFocus.TELEMETRY:init_cmd_str = f"pipelex init {focus}"console.print(f"[dim]You can initialize later by running:[/dim] [cyan]{init_cmd_str}[/cyan]")console.print()@@ -298,9 +148,11 @@ def execute_initialization(console: Console,needs_config: bool,needs_inference: bool,+ needs_routing: bool,needs_telemetry: bool,reset: bool,check_inference: bool,+ check_routing: bool,backends_toml_path: str,telemetry_config_path: str,):@@ -310,9 +162,11 @@ def execute_initialization(console: Rich Console instance for output.needs_config: Whether to initialize config files.needs_inference: Whether to set up inference backends.+ needs_routing: Whether to set up routing profiles.needs_telemetry: Whether to set up telemetry.reset: Whether this is a reset operation.check_inference: Whether inference was in focus.+ check_routing: Whether routing was in focus.backends_toml_path: Path to backends.toml file.telemetry_config_path: Path to telemetry config file.@@ -337,6 +191,21 @@ def execute_initialization(console.print()customize_backends_config()+ # Automatically set up routing after backends (unless routing is the specific focus)+ if not check_routing:+ selected_backend_keys = get_selected_backend_keys(backends_toml_path)+ if selected_backend_keys:+ customize_routing_profile(selected_backend_keys)++ # Step 2.5: Set up routing profile if specifically requested+ if needs_routing:+ console.print()+ selected_backend_keys = get_selected_backend_keys(backends_toml_path)+ if selected_backend_keys:+ customize_routing_profile(selected_backend_keys)+ else:+ console.print("[yellow]⚠ Warning: No backends enabled. Please run 'pipelex init inference' first.[/yellow]")+# Step 3: Set up telemetry if neededif needs_telemetry:telemetry_mode = setup_telemetry(console, telemetry_config_path)@@ -351,10 +220,10 @@ def init_cmd(skip_confirmation: bool = False,silent: bool = False,):- """Initialize Pipelex configuration, inference backends, and telemetry if needed, in a unified flow.+ """Initialize Pipelex configuration, inference backends, routing, and telemetry if needed, in a unified flow.Args:- focus: What to initialize - 'config', 'inference', 'telemetry', or 'all' (default)+ focus: What to initialize - 'config', 'inference', 'routing', 'telemetry', or 'all' (default)reset: Whether to reset/overwrite existing filesskip_confirmation: If True, skip the confirmation prompt (used when called from doctor --fix)silent: If True, suppress all output when everything is already configured@@ -363,19 +232,23 @@ def init_cmd(pipelex_config_dir = config_manager.pipelex_config_dirtelemetry_config_path = os.path.join(pipelex_config_dir, TELEMETRY_CONFIG_FILE_NAME)backends_toml_path = os.path.join(pipelex_config_dir, "inference", "backends.toml")+ routing_profiles_toml_path = os.path.join(pipelex_config_dir, "inference", "routing_profiles.toml")# Determine what to check based on focus parametercheck_config = focus in (InitFocus.ALL, InitFocus.CONFIG)check_inference = focus in (InitFocus.ALL, InitFocus.INFERENCE)+ check_routing = focus == InitFocus.ROUTINGcheck_telemetry = focus in (InitFocus.ALL, InitFocus.TELEMETRY)# Check what needs to be initialized- needs_config, needs_inference, needs_telemetry = determine_needs(+ needs_config, needs_inference, needs_routing, needs_telemetry = determine_needs(reset=reset,check_config=check_config,check_inference=check_inference,+ check_routing=check_routing,check_telemetry=check_telemetry,backends_toml_path=backends_toml_path,+ routing_profiles_toml_path=routing_profiles_toml_path,telemetry_config_path=telemetry_config_path,)@@ -383,14 +256,14 @@ def init_cmd(user_already_confirmed = False# If nothing needs to be done, handle based on focus- if not needs_config and not needs_inference and not needs_telemetry:+ if not needs_config and not needs_inference and not needs_routing and not needs_telemetry:# In silent mode, just return without any outputif silent:return- if handle_already_configured(focus, console, backends_toml_path, telemetry_config_path):+ if handle_already_configured(focus, console, backends_toml_path, routing_profiles_toml_path, telemetry_config_path):# User wants to reconfigure- needs_config, needs_inference, needs_telemetry = update_needs_for_reconfigure(focus)+ needs_config, needs_inference, needs_routing, needs_telemetry = update_needs_for_reconfigure(focus)user_already_confirmed = Trueelse:# User doesn't want to reconfigure, exit@@ -405,6 +278,7 @@ def init_cmd(console=console,needs_config=needs_config,needs_inference=needs_inference,+ needs_routing=needs_routing,needs_telemetry=needs_telemetry,reset=reset,focus=focus,@@ -418,9 +292,11 @@ def init_cmd(console=console,needs_config=needs_config,needs_inference=needs_inference,+ needs_routing=needs_routing,needs_telemetry=needs_telemetry,reset=reset,check_inference=check_inference,+ check_routing=check_routing,backends_toml_path=backends_toml_path,telemetry_config_path=telemetry_config_path,)diff --git a/pipelex/cli/commands/init/config_files.py b/pipelex/cli/commands/init/config_files.pynew file mode 100644index 000000000..c565bbd86--- /dev/null+++ b/pipelex/cli/commands/init/config_files.py@@ -0,0 +1,78 @@+"""Configuration files management for the init command."""++import os+import shutil++import typer++from pipelex.exceptions import PipelexCLIError+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.system.configuration.config_loader import config_manager+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME+++def init_config(reset: bool = False, dry_run: bool = False) -> int:+ """Initialize pipelex configuration in the .pipelex directory. Does not install telemetry, just the main config dans inference backends.++ Args:+ reset: Whether to overwrite existing files.+ dry_run: Whether to only print the files that would be copied, without actually copying them.++ Returns:+ The number of files copied.+ """+ config_template_dir = str(get_kit_configs_dir())+ target_config_dir = config_manager.pipelex_config_dir++ os.makedirs(target_config_dir, exist_ok=True)++ try:+ copied_files: list[str] = []+ existing_files: list[str] = []++ def copy_directory_structure(src_dir: str, dst_dir: str, relative_path: str = "", dry_run: bool = False) -> None:+ """Recursively copy directory structure, handling existing files."""+ for item in os.listdir(src_dir):+ src_item = os.path.join(src_dir, item)+ dst_item = os.path.join(dst_dir, item)+ relative_item = os.path.join(relative_path, item) if relative_path else item++ # Skip telemetry.toml - it will be created when user is prompted+ if item == TELEMETRY_CONFIG_FILE_NAME:+ continue++ if os.path.isdir(src_item):+ if not dry_run:+ os.makedirs(dst_item, exist_ok=True)+ copy_directory_structure(src_item, dst_item, relative_item, dry_run)+ elif os.path.exists(dst_item) and not reset:+ existing_files.append(relative_item)+ else:+ if not dry_run:+ shutil.copy2(src_item, dst_item)+ copied_files.append(relative_item)++ copy_directory_structure(src_dir=config_template_dir, dst_dir=target_config_dir, dry_run=dry_run)++ if dry_run:+ return len(copied_files)++ # Report results+ if copied_files:+ typer.echo(f"✅ Copied {len(copied_files)} files to {target_config_dir}:")+ for file in sorted(copied_files):+ typer.echo(f" • {file}")++ if existing_files:+ typer.echo(f"ℹ️ Skipped {len(existing_files)} existing files (use --reset to overwrite):")+ for file in sorted(existing_files):+ typer.echo(f" • {file}")++ if not copied_files and not existing_files:+ typer.echo(f"✅ Configuration directory {target_config_dir} is already up to date")++ except Exception as exc:+ msg = f"Failed to initialize configuration: {exc}"+ raise PipelexCLIError(msg) from exc++ return len(copied_files)diff --git a/pipelex/cli/commands/init/routing.py b/pipelex/cli/commands/init/routing.pynew file mode 100644index 000000000..cb0504e17--- /dev/null+++ b/pipelex/cli/commands/init/routing.py@@ -0,0 +1,153 @@+"""Routing profile configuration logic for the init command."""++import os+from typing import Any++from rich.console import Console+from rich.prompt import Confirm+from tomlkit import table++from pipelex.cli.commands.init.ui.backends_ui import get_backend_options_from_toml+from pipelex.cli.commands.init.ui.routing_ui import (+ build_fallback_order_panel,+ build_primary_backend_panel,+ display_routing_profile_result,+ prompt_fallback_order,+ prompt_primary_backend,+)+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.system.configuration.config_loader import config_manager+from pipelex.tools.misc.file_utils import path_exists+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path+++def customize_routing_profile(selected_backend_keys: list[str]) -> None:+ """Interactively customize routing profile based on selected backends.++ Args:+ selected_backend_keys: List of backend keys that are enabled.+ """+ console = Console()+ routing_profiles_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "routing_profiles.toml")+ template_routing_path = os.path.join(str(get_kit_configs_dir()), "inference", "routing_profiles.toml")+ backends_toml_path = os.path.join(config_manager.pipelex_config_dir, "inference", "backends.toml")++ if not path_exists(routing_profiles_toml_path):+ console.print("[yellow]⚠ Warning: routing_profiles.toml not found, skipping routing customization[/yellow]")+ return++ try:+ # Load template for reference (to check if profiles exist)+ template_toml = load_toml_with_tomlkit(template_routing_path)++ # Load the user's routing_profiles.toml file+ toml_doc = load_toml_with_tomlkit(routing_profiles_toml_path)++ # Get backend options for display names+ template_backends_path = os.path.join(str(get_kit_configs_dir()), "inference", "backends.toml")+ backend_options = get_backend_options_from_toml(template_backends_path, backends_toml_path)++ # Case 1: pipelex_inference is enabled - keep pipelex_first+ if "pipelex_inference" in selected_backend_keys:+ toml_doc["active"] = "pipelex_first"+ save_toml_to_path(toml_doc, routing_profiles_toml_path)+ display_routing_profile_result(console, "pipelex_first", created=False)+ return++ # Case 2: Only one backend selected - use all_{backend_key} profile+ if len(selected_backend_keys) == 1:+ backend_key = selected_backend_keys[0]+ profile_name = f"all_{backend_key}"++ # Check if profile exists in template+ template_profiles: dict[str, Any] = template_toml.get("profiles") or {} # type: ignore[assignment]+ profile_exists = profile_name in template_profiles++ if not profile_exists:+ # Ask for confirmation to create the profile+ console.print()+ console.print(f"[yellow]Profile {profile_name!r} does not exist in the template.[/yellow]")+ if not Confirm.ask("[bold]Would you like to create it?[/bold]", default=True):+ console.print("[dim]Keeping current routing profile configuration.[/dim]")+ return++ # Create the profile+ if "profiles" not in toml_doc:+ toml_doc["profiles"] = {}++ # Use a dict for the profile data+ profile_data = {+ "description": f"Use {backend_key} backend for all its supported models",+ "default": backend_key,+ }+ toml_doc["profiles"][profile_name] = profile_data # type: ignore[index]++ # Set as active profile+ toml_doc["active"] = profile_name+ save_toml_to_path(toml_doc, routing_profiles_toml_path)+ display_routing_profile_result(console, profile_name, created=not profile_exists)+ return++ # Case 3: Multiple backends (no pipelex_inference) - create/update custom_routing profile+ console.print()+ console.print("[cyan]Setting up routing for multiple backends...[/cyan]")+ console.print()++ # Show panel and prompt for primary backend+ console.print(build_primary_backend_panel(selected_backend_keys, backend_options))+ primary_backend = prompt_primary_backend(console, selected_backend_keys)++ # Prompt for fallback order if there are 2+ remaining backends+ remaining_backends = [b for b in selected_backend_keys if b != primary_backend]+ fallback_order: list[str] | None = None++ if len(remaining_backends) >= 2:+ console.print()+ console.print(build_fallback_order_panel(remaining_backends, backend_options))+ # Get the ordered remaining backends from user (default keeps current order)+ ordered_remaining = prompt_fallback_order(console, remaining_backends, backend_options)+ fallback_order = [primary_backend, *ordered_remaining]+ elif len(remaining_backends) == 1:+ # Only one remaining backend - set fallback_order with primary first+ fallback_order = [primary_backend, *remaining_backends]+ else:+ # Only primary backend selected - no fallback needed+ fallback_order = None++ # Create or update custom_routing profile+ if "profiles" not in toml_doc:+ toml_doc["profiles"] = {}++ # Insert custom_routing at the beginning of profiles (after standard profiles if they exist)+ # We'll create a new profiles dict with custom_routing first+ new_profiles = table()++ # Create custom_routing profile with nested routes table+ custom_routing_profile = table()+ custom_routing_profile["description"] = "Custom routing"+ custom_routing_profile["default"] = primary_backend+ if fallback_order:+ custom_routing_profile["fallback_order"] = fallback_order+ custom_routing_profile["routes"] = table()+ new_profiles["custom_routing"] = custom_routing_profile++ # Copy existing profiles after custom_routing+ existing_profiles: dict[str, Any] = toml_doc.get("profiles") or {} # type: ignore[assignment]+ for profile_key, profile_value in existing_profiles.items(): # type: ignore[union-attr]+ if profile_key != "custom_routing":+ new_profiles[profile_key] = profile_value++ toml_doc["profiles"] = new_profiles # type: ignore[assignment]+ toml_doc["active"] = "custom_routing"++ save_toml_to_path(toml_doc, routing_profiles_toml_path)+ display_routing_profile_result(console, "custom_routing", created=True)++ # Inform user about customization options+ console.print("[dim]You can find your custom routing profile in:[/dim]")+ console.print(f"[dim] {routing_profiles_toml_path}[/dim]")+ console.print("[dim]You can further customize which models get used on which backend by editing the routes section.[/dim]")++ except Exception as exc:+ console.print(f"[yellow]⚠ Warning: Failed to customize routing profile: {exc}[/yellow]")+ console.print("[dim]You can manually edit .pipelex/inference/routing_profiles.toml later[/dim]")diff --git a/pipelex/cli/commands/init/telemetry.py b/pipelex/cli/commands/init/telemetry.pynew file mode 100644index 000000000..cfe26ea2e--- /dev/null+++ b/pipelex/cli/commands/init/telemetry.py@@ -0,0 +1,39 @@+"""Telemetry configuration logic for the init command."""++import os++from rich.console import Console++from pipelex.cli.commands.init.ui.telemetry_ui import build_telemetry_selection_panel, prompt_telemetry_mode+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryMode+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path+++def setup_telemetry(console: Console, telemetry_config_path: str) -> TelemetryMode:+ """Set up telemetry configuration interactively.++ Args:+ console: Rich Console instance for user interaction.+ telemetry_config_path: Path to save the telemetry configuration.++ Returns:+ The selected TelemetryMode.++ Raises:+ typer.Exit: If user chooses to quit.+ """+ console.print()+ console.print(build_telemetry_selection_panel())++ telemetry_mode = prompt_telemetry_mode(console)++ # Save telemetry config+ template_path = os.path.join(str(get_kit_configs_dir()), TELEMETRY_CONFIG_FILE_NAME)+ toml_doc = load_toml_with_tomlkit(template_path)+ toml_doc["telemetry_mode"] = telemetry_mode+ save_toml_to_path(toml_doc, telemetry_config_path)++ console.print(f"\n[green]✓[/green] Telemetry mode set to: [bold cyan]{telemetry_mode}[/bold cyan]")++ return telemetry_modediff --git a/pipelex/cli/commands/init/ui/__init__.py b/pipelex/cli/commands/init/ui/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/cli/commands/init_ui.py b/pipelex/cli/commands/init/ui/backends_ui.pysimilarity index 55%rename from pipelex/cli/commands/init_ui.pyrename to pipelex/cli/commands/init/ui/backends_ui.pyindex 309d15724..c7fa48774 100644--- a/pipelex/cli/commands/init_ui.py+++ b/pipelex/cli/commands/init/ui/backends_ui.py@@ -1,23 +1,15 @@-"""UI components for the init command.--This module contains all user interface logic for the Pipelex initialization process,-including prompts, panels, and user input validation.-"""--from __future__ import annotations+"""UI components for backend configuration in the init command."""import typerfrom rich.console import Console, Groupfrom rich.panel import Panel-from rich.prompt import Confirm, Prompt+from rich.prompt import Promptfrom rich.table import Tablefrom rich.text import Text-from pipelex.system.telemetry.telemetry_config import TelemetryModefrom pipelex.tools.misc.file_utils import path_existsfrom pipelex.tools.misc.string_utils import snake_to_capitalize_first_letterfrom pipelex.tools.misc.toml_utils import load_toml_from_path-from pipelex.types import StrEnumdef get_backend_options_from_toml(template_path: str, existing_path: str | None = None) -> list[tuple[str, str]]:@@ -100,15 +92,6 @@ def get_currently_enabled_backends(backends_toml_path: str, backend_options: lisreturn sorted(currently_enabled)-class InitFocus(StrEnum):- """Focus options for initialization."""-- ALL = "all"- CONFIG = "config"- INFERENCE = "inference"- TELEMETRY = "telemetry"--def build_backend_selection_panel(backend_options: list[tuple[str, str]], currently_enabled: list[int] | None = None) -> Panel:"""Create a Rich Panel for backend selection with options table.@@ -248,190 +231,3 @@ def display_selected_backends(console: Console, selected_indices: list[int], bacconsole.print(f"\n[green]✓[/green] Configured {len(selected_names)} backend(s):")for name in selected_names:console.print(f" • {name}")---def build_telemetry_selection_panel() -> Panel:- """Create a Rich Panel for telemetry mode selection.-- Returns:- A Panel containing the telemetry selection interface.- """- table = Table(show_header=False, box=None, padding=(0, 2))- table.add_column(style="bold cyan", justify="right")- table.add_column(style="bold")- table.add_column()-- table.add_row("[1]", TelemetryMode.OFF, "No telemetry data collected")- table.add_row("[2]", TelemetryMode.ANONYMOUS, "Anonymous usage data only")- table.add_row("[3]", TelemetryMode.IDENTIFIED, "Usage data with user identification")- table.add_row("[Q]", "[dim]quit[/dim]", "[dim]Exit without configuring[/dim]")-- description = Text(- "Pipelex can collect anonymous usage data to help improve the product.",- style="dim",- )-- return Panel(- Group(description, Text(""), table),- title="[bold yellow]Telemetry Configuration[/bold yellow]",- border_style="yellow",- padding=(1, 2),- )---def prompt_telemetry_mode(console: Console) -> TelemetryMode:- """Prompt user to select telemetry mode with validation.-- Args:- console: Rich Console instance for user interaction.-- Returns:- Selected TelemetryMode.-- Raises:- typer.Exit: If user chooses to quit.- """- # Map choice to telemetry mode- mode_map: dict[str, TelemetryMode] = {- "1": TelemetryMode.OFF,- "2": TelemetryMode.ANONYMOUS,- "3": TelemetryMode.IDENTIFIED,- "off": TelemetryMode.OFF,- "anonymous": TelemetryMode.ANONYMOUS,- "identified": TelemetryMode.IDENTIFIED,- }-- # Loop until valid input- telemetry_mode: TelemetryMode | None = None- while telemetry_mode is None:- choice_str = Prompt.ask("[bold]Enter your choice[/bold]", console=console)- choice_input = choice_str.lower().strip()-- # Handle quit option- if choice_input in ("q", "quit"):- console.print("\n[yellow]Exiting without configuring telemetry.[/yellow]")- raise typer.Exit(code=0)-- if choice_input in mode_map:- telemetry_mode = mode_map[choice_input]- else:- console.print(- f"[red]Invalid choice: '{choice_str}'.[/red] "- "Please enter [cyan]1[/cyan], [cyan]2[/cyan], [cyan]3[/cyan], or [cyan]q[/cyan] to quit.\n"- )-- return telemetry_mode---def display_already_configured_message(focus: InitFocus, console: Console, config_path: str) -> bool:- """Display 'already configured' message and ask if user wants to reconfigure.-- Args:- focus: The initialization focus area.- console: Rich Console instance for output.- config_path: Path to the configuration file.-- Returns:- True if user wants to reconfigure, False otherwise.- """- # Mapping of focus to (subject, action_verb)- focus_messages = {- InitFocus.INFERENCE: ("Inference backends", "inference backends"),- InitFocus.TELEMETRY: ("Telemetry preferences", "telemetry preferences"),- InitFocus.CONFIG: ("Configuration files", "configuration"),- }-- if focus == InitFocus.ALL:- console.print()- console.print("[green]✓[/green] Pipelex is already fully initialized!")- console.print()- console.print("[dim]Configuration files are in place:[/dim] [cyan].pipelex/[/cyan]")- console.print("[dim]Telemetry preferences are configured[/dim]")- console.print()- console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")- console.print(" [cyan]pipelex init --reset[/cyan]")- console.print()- return False-- if focus == InitFocus.CONFIG:- console.print()- console.print("[green]✓[/green] Configuration files are already in place!")- console.print()- console.print("[dim]Configuration directory:[/dim] [cyan].pipelex/[/cyan]")- console.print()- console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")- console.print(f" [cyan]pipelex init {focus} --reset[/cyan]")- console.print()- return False-- if focus in focus_messages:- subject, action_verb = focus_messages[focus]- console.print()- console.print(f"[green]✓[/green] {subject} are already configured!")- console.print()- console.print(f"[dim]Configuration file:[/dim] [cyan]{config_path}[/cyan]")- console.print()-- return Confirm.ask(f"[bold]Would you like to reconfigure {action_verb}?[/bold]", default=False)-- return False---def build_initialization_panel(needs_config: bool, needs_inference: bool, needs_telemetry: bool, reset: bool) -> Panel:- """Build the initialization confirmation panel.-- Args:- needs_config: Whether config initialization is needed.- needs_inference: Whether inference setup is needed.- needs_telemetry: Whether telemetry setup is needed.- reset: Whether this is a reset operation.-- Returns:- A Panel containing the initialization confirmation message.- """- # Build message based on what's being initialized- message_parts: list[str] = []- if reset:- if needs_config:- message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")- if needs_inference:- message_parts.append("• [yellow]Reset and reconfigure[/yellow] inference backends")- if needs_telemetry:- message_parts.append("• [yellow]Reset and reconfigure[/yellow] telemetry preferences")- else:- if needs_config:- message_parts.append("• Create required configuration files in [cyan].pipelex/[/cyan]")- if needs_inference:- message_parts.append("• Ask you to choose your inference backends")- if needs_telemetry:- message_parts.append("• Ask you to choose your telemetry preferences")-- # Determine title based on what's being initialized- num_items = sum([needs_config, needs_inference, needs_telemetry])- if reset:- if num_items > 1:- title_text = "[bold yellow]Resetting Configuration[/bold yellow]"- elif needs_config:- title_text = "[bold yellow]Resetting Configuration Files[/bold yellow]"- elif needs_inference:- title_text = "[bold yellow]Resetting Inference Backends[/bold yellow]"- else:- title_text = "[bold yellow]Resetting Telemetry[/bold yellow]"- elif num_items > 1:- title_text = "[bold cyan]Pipelex Initialization[/bold cyan]"- elif needs_config:- title_text = "[bold cyan]Configuration Setup[/bold cyan]"- elif needs_inference:- title_text = "[bold cyan]Inference Backend Setup[/bold cyan]"- else:- title_text = "[bold cyan]Telemetry Setup[/bold cyan]"-- message = "\n".join(message_parts)- border_color = "yellow" if reset else "cyan"-- return Panel(- message,- title=title_text,- border_style=border_color,- padding=(1, 2),- )diff --git a/pipelex/cli/commands/init/ui/general_ui.py b/pipelex/cli/commands/init/ui/general_ui.pynew file mode 100644index 000000000..eedc5c474--- /dev/null+++ b/pipelex/cli/commands/init/ui/general_ui.py@@ -0,0 +1,134 @@+"""General UI components for the init command."""++from rich.console import Console+from rich.panel import Panel+from rich.prompt import Confirm++from pipelex.cli.commands.init.ui.types import InitFocus+++def display_already_configured_message(focus: InitFocus, console: Console, config_path: str) -> bool:+ """Display 'already configured' message and ask if user wants to reconfigure.++ Args:+ focus: The initialization focus area.+ console: Rich Console instance for output.+ config_path: Path to the configuration file.++ Returns:+ True if user wants to reconfigure, False otherwise.+ """+ # Mapping of focus to (subject, action_verb)+ focus_messages = {+ InitFocus.INFERENCE: ("Inference backends", "inference backends"),+ InitFocus.ROUTING: ("Routing profile", "routing profile"),+ InitFocus.TELEMETRY: ("Telemetry preferences", "telemetry preferences"),+ InitFocus.CONFIG: ("Configuration files", "configuration"),+ }++ if focus == InitFocus.ALL:+ console.print()+ console.print("[green]✓[/green] Pipelex is already fully initialized!")+ console.print()+ console.print("[dim]Configuration files are in place:[/dim] [cyan].pipelex/[/cyan]")+ console.print("[dim]Telemetry preferences are configured[/dim]")+ console.print()+ console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")+ console.print(" [cyan]pipelex init --reset[/cyan]")+ console.print()+ return False++ if focus == InitFocus.CONFIG:+ console.print()+ console.print("[green]✓[/green] Configuration files are already in place!")+ console.print()+ console.print("[dim]Configuration directory:[/dim] [cyan].pipelex/[/cyan]")+ console.print()+ console.print("[dim]💡 Tip: Use[/dim] [cyan]--reset[/cyan] [dim]to reconfigure or troubleshoot:[/dim]")+ console.print(f" [cyan]pipelex init {focus} --reset[/cyan]")+ console.print()+ return False++ if focus in focus_messages:+ subject, action_verb = focus_messages[focus]+ console.print()+ if focus == InitFocus.ROUTING:+ console.print(f"[green]✓[/green] {subject} is already configured!")+ else:+ console.print(f"[green]✓[/green] {subject} are already configured!")+ console.print()+ console.print(f"[dim]Configuration file:[/dim] [cyan]{config_path}[/cyan]")+ console.print()++ return Confirm.ask(f"[bold]Would you like to reconfigure {action_verb}?[/bold]", default=False)++ return False+++def build_initialization_panel(needs_config: bool, needs_inference: bool, needs_routing: bool, needs_telemetry: bool, reset: bool) -> Panel:+ """Build the initialization confirmation panel.++ Args:+ needs_config: Whether config initialization is needed.+ needs_inference: Whether inference setup is needed.+ needs_routing: Whether routing setup is needed.+ needs_telemetry: Whether telemetry setup is needed.+ reset: Whether this is a reset operation.++ Returns:+ A Panel containing the initialization confirmation message.+ """+ # Build message based on what's being initialized+ message_parts: list[str] = []+ if reset:+ if needs_config:+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")+ if needs_inference:+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] inference backends")+ if needs_routing:+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] routing profile")+ if needs_telemetry:+ message_parts.append("• [yellow]Reset and reconfigure[/yellow] telemetry preferences")+ else:+ if needs_config:+ message_parts.append("• Create required configuration files in [cyan].pipelex/[/cyan]")+ if needs_inference:+ message_parts.append("• Ask you to choose your inference backends")+ if needs_routing:+ message_parts.append("• Ask you to configure your routing profile")+ if needs_telemetry:+ message_parts.append("• Ask you to choose your telemetry preferences")++ # Determine title based on what's being initialized+ num_items = sum([needs_config, needs_inference, needs_routing, needs_telemetry])+ if reset:+ if num_items > 1:+ title_text = "[bold yellow]Resetting Configuration[/bold yellow]"+ elif needs_config:+ title_text = "[bold yellow]Resetting Configuration Files[/bold yellow]"+ elif needs_inference:+ title_text = "[bold yellow]Resetting Inference Backends[/bold yellow]"+ elif needs_routing:+ title_text = "[bold yellow]Resetting Routing Profile[/bold yellow]"+ else:+ title_text = "[bold yellow]Resetting Telemetry[/bold yellow]"+ elif num_items > 1:+ title_text = "[bold cyan]Pipelex Initialization[/bold cyan]"+ elif needs_config:+ title_text = "[bold cyan]Configuration Setup[/bold cyan]"+ elif needs_inference:+ title_text = "[bold cyan]Inference Backend Setup[/bold cyan]"+ elif needs_routing:+ title_text = "[bold cyan]Routing Profile Setup[/bold cyan]"+ else:+ title_text = "[bold cyan]Telemetry Setup[/bold cyan]"++ message = "\n".join(message_parts)+ border_color = "yellow" if reset else "cyan"++ return Panel(+ message,+ title=title_text,+ border_style=border_color,+ padding=(1, 2),+ )diff --git a/pipelex/cli/commands/init/ui/routing_ui.py b/pipelex/cli/commands/init/ui/routing_ui.pynew file mode 100644index 000000000..082f29ddd--- /dev/null+++ b/pipelex/cli/commands/init/ui/routing_ui.py@@ -0,0 +1,207 @@+"""UI components for routing profile configuration in the init command."""++from rich.console import Console, Group+from rich.panel import Panel+from rich.prompt import Prompt+from rich.table import Table+from rich.text import Text++from pipelex.tools.misc.string_utils import snake_to_capitalize_first_letter+++def build_primary_backend_panel(backend_keys: list[str], backend_options: list[tuple[str, str]]) -> Panel:+ """Build a panel for selecting a primary backend from enabled backends.++ Args:+ backend_keys: List of enabled backend keys.+ backend_options: List of all backend options to get display names.++ Returns:+ A Panel containing the primary backend selection interface.+ """+ # Create a mapping of backend_key to display_name+ backend_key_to_name = dict(backend_options)++ table = Table(show_header=False, box=None, padding=(0, 2))+ table.add_column(style="bold cyan", justify="right", width=4)+ table.add_column(style="bold", width=30)++ for idx, backend_key in enumerate(backend_keys, start=1):+ display_name = backend_key_to_name.get(backend_key, snake_to_capitalize_first_letter(backend_key))+ table.add_row(f"[{idx}]", display_name)++ description = Text(+ "Multiple backends are enabled. Select which backend should be your primary/default.\n"+ "The primary backend will be used for models that don't have an exact match or pattern match in your routes.",+ style="dim",+ )++ return Panel(+ Group(description, Text(""), table),+ title="[bold yellow]Primary Backend Selection[/bold yellow]",+ border_style="yellow",+ padding=(1, 2),+ )+++def prompt_primary_backend(console: Console, backend_keys: list[str]) -> str:+ """Prompt user to select a primary backend from enabled backends.++ Args:+ console: Rich Console instance for user interaction.+ backend_keys: List of enabled backend keys.++ Returns:+ The selected backend key.+ """+ # Default to first backend+ default_str = "1"++ selected_backend: str | None = None+ while selected_backend is None:+ choice_str = Prompt.ask("[bold]Enter your choice[/bold]", default=default_str, console=console)+ choice_input = choice_str.strip()++ # Parse input - handle empty (use default)+ if not choice_input:+ selected_backend = backend_keys[0]+ break++ try:+ # Parse as 1-based index from user input+ user_index = int(choice_input)++ # Validate index is in range (1-based)+ if user_index < 1 or user_index > len(backend_keys):+ max_idx = len(backend_keys)+ console.print(f"[red]Invalid choice: {user_index}.[/red] Please enter a number between 1 and {max_idx}.\n")+ continue++ # Convert to 0-based index and get backend key+ selected_backend = backend_keys[user_index - 1]+ break++ except ValueError:+ console.print(f"[red]Invalid input: '{choice_str}'.[/red] Please enter a number.\n")++ return selected_backend+++def build_fallback_order_panel(remaining_backends: list[str], backend_options: list[tuple[str, str]]) -> Panel:+ """Build a panel explaining fallback order configuration.++ Args:+ remaining_backends: List of backend keys (excluding primary).+ backend_options: List of all backend options to get display names.++ Returns:+ A Panel containing the fallback order explanation and current order.+ """+ # Create a mapping of backend_key to display_name+ backend_key_to_name = dict(backend_options)++ table = Table(show_header=False, box=None, padding=(0, 2))+ table.add_column(style="bold cyan", justify="right", width=4)+ table.add_column(style="bold", width=30)++ for idx, backend_key in enumerate(remaining_backends, start=1):+ display_name = backend_key_to_name.get(backend_key, snake_to_capitalize_first_letter(backend_key))+ table.add_row(f"[{idx}]", display_name)++ description = Text(+ "Fallback order determines which backends to try when a model has no exact match or pattern match\n"+ "and is not available in your default backend.\nCurrent order:",+ style="dim",+ )++ return Panel(+ Group(description, Text(""), table),+ title="[bold yellow]Fallback Backend Order[/bold yellow]",+ border_style="yellow",+ padding=(1, 2),+ )+++def prompt_fallback_order(console: Console, remaining_backends: list[str], backend_options: list[tuple[str, str]]) -> list[str]:+ """Prompt user to order the remaining backends for fallback.++ Args:+ console: Rich Console instance for user interaction.+ remaining_backends: List of backend keys (excluding primary).+ backend_options: List of all backend options to get display names.++ Returns:+ Ordered list of backend keys for fallback.+ """+ backend_key_to_name = dict(backend_options)++ console.print()+ console.print("[dim]Enter the order of backends (space or comma separated indices).[/dim]")+ console.print("[dim]For example: '2 1 3' or '2,1,3' to reorder them.[/dim]")+ console.print()++ fallback_order: list[str] | None = None+ while fallback_order is None:+ # Show current order as default+ default_order = " ".join(str(i + 1) for i in range(len(remaining_backends)))+ choice_str = Prompt.ask("[bold]Enter order[/bold]", default=default_order, console=console)+ choice_input = choice_str.strip()++ # Handle empty input (use default order)+ if not choice_input:+ fallback_order = remaining_backends.copy()+ break++ # Split by comma or space+ parts = choice_input.replace(",", " ").split()++ try:+ # Parse as 1-based indices from user input+ user_indices = [int(part.strip()) for part in parts if part.strip()]++ # Validate: must have exactly the same number of indices+ if len(user_indices) != len(remaining_backends):+ console.print(f"[red]Error: You must specify all {len(remaining_backends)} backends.[/red] You provided {len(user_indices)}.\n")+ continue++ # Validate: all indices are in range (1-based)+ invalid_indices = [i for i in user_indices if i < 1 or i > len(remaining_backends)]+ if invalid_indices:+ max_idx = len(remaining_backends)+ console.print(f"[red]Invalid choice(s): {invalid_indices}.[/red] Please enter numbers between 1 and {max_idx}.\n")+ continue++ # Validate: no duplicates+ if len(user_indices) != len(set(user_indices)):+ console.print("[red]Error: Duplicate indices not allowed.[/red] Each backend must appear exactly once.\n")+ continue++ # Convert to 0-based indices and reorder backends+ zero_based_indices = [i - 1 for i in user_indices]+ fallback_order = [remaining_backends[i] for i in zero_based_indices]+ break++ except ValueError:+ console.print(f"[red]Invalid input: '{choice_str}'.[/red] Please enter numbers separated by commas or spaces.\n")++ # Display the final order+ console.print("\n[green]✓[/green] Fallback order set to:")+ for idx, backend_key in enumerate(fallback_order, start=1):+ display_name = backend_key_to_name.get(backend_key, snake_to_capitalize_first_letter(backend_key))+ console.print(f" {idx}. {display_name}")++ return fallback_order+++def display_routing_profile_result(console: Console, profile_name: str, created: bool = False) -> None:+ """Display confirmation of routing profile configuration.++ Args:+ console: Rich Console instance for output.+ profile_name: Name of the routing profile that was set.+ created: Whether the profile was newly created.+ """+ if created:+ console.print(f"\n[green]✓[/green] Created and set routing profile to: [bold cyan]{profile_name}[/bold cyan]")+ else:+ console.print(f"\n[green]✓[/green] Routing profile set to: [bold cyan]{profile_name}[/bold cyan]")diff --git a/pipelex/cli/commands/init/ui/telemetry_ui.py b/pipelex/cli/commands/init/ui/telemetry_ui.pynew file mode 100644index 000000000..0a338d74d--- /dev/null+++ b/pipelex/cli/commands/init/ui/telemetry_ui.py@@ -0,0 +1,83 @@+"""UI components for telemetry configuration in the init command."""++import typer+from rich.console import Console, Group+from rich.panel import Panel+from rich.prompt import Prompt+from rich.table import Table+from rich.text import Text++from pipelex.system.telemetry.telemetry_config import TelemetryMode+++def build_telemetry_selection_panel() -> Panel:+ """Create a Rich Panel for telemetry mode selection.++ Returns:+ A Panel containing the telemetry selection interface.+ """+ table = Table(show_header=False, box=None, padding=(0, 2))+ table.add_column(style="bold cyan", justify="right")+ table.add_column(style="bold")+ table.add_column()++ table.add_row("[1]", TelemetryMode.OFF, "No telemetry data collected")+ table.add_row("[2]", TelemetryMode.ANONYMOUS, "Anonymous usage data only")+ table.add_row("[3]", TelemetryMode.IDENTIFIED, "Usage data with user identification")+ table.add_row("[Q]", "[dim]quit[/dim]", "[dim]Exit without configuring[/dim]")++ description = Text(+ "Pipelex can collect anonymous usage data to help improve the product.",+ style="dim",+ )++ return Panel(+ Group(description, Text(""), table),+ title="[bold yellow]Telemetry Configuration[/bold yellow]",+ border_style="yellow",+ padding=(1, 2),+ )+++def prompt_telemetry_mode(console: Console) -> TelemetryMode:+ """Prompt user to select telemetry mode with validation.++ Args:+ console: Rich Console instance for user interaction.++ Returns:+ Selected TelemetryMode.++ Raises:+ typer.Exit: If user chooses to quit.+ """+ # Map choice to telemetry mode+ mode_map: dict[str, TelemetryMode] = {+ "1": TelemetryMode.OFF,+ "2": TelemetryMode.ANONYMOUS,+ "3": TelemetryMode.IDENTIFIED,+ "off": TelemetryMode.OFF,+ "anonymous": TelemetryMode.ANONYMOUS,+ "identified": TelemetryMode.IDENTIFIED,+ }++ # Loop until valid input+ telemetry_mode: TelemetryMode | None = None+ while telemetry_mode is None:+ choice_str = Prompt.ask("[bold]Enter your choice[/bold]", console=console)+ choice_input = choice_str.lower().strip()++ # Handle quit option+ if choice_input in ("q", "quit"):+ console.print("\n[yellow]Exiting without configuring telemetry.[/yellow]")+ raise typer.Exit(code=0)++ if choice_input in mode_map:+ telemetry_mode = mode_map[choice_input]+ else:+ console.print(+ f"[red]Invalid choice: '{choice_str}'.[/red] "+ "Please enter [cyan]1[/cyan], [cyan]2[/cyan], [cyan]3[/cyan], or [cyan]q[/cyan] to quit.\n"+ )++ return telemetry_modediff --git a/pipelex/cli/commands/init/ui/types.py b/pipelex/cli/commands/init/ui/types.pynew file mode 100644index 000000000..274ce2115--- /dev/null+++ b/pipelex/cli/commands/init/ui/types.py@@ -0,0 +1,13 @@+"""Types for the init command UI."""++from pipelex.types import StrEnum+++class InitFocus(StrEnum):+ """Focus options for initialization."""++ ALL = "all"+ CONFIG = "config"+ INFERENCE = "inference"+ ROUTING = "routing"+ TELEMETRY = "telemetry"diff --git a/pipelex/cli/commands/run_cmd.py b/pipelex/cli/commands/run_cmd.pyindex 8572f9050..f3e804054 100644--- a/pipelex/cli/commands/run_cmd.py+++ b/pipelex/cli/commands/run_cmd.py@@ -6,13 +6,28 @@ from typing import Annotatedimport clickimport typer-from posthog import new_context, tag+from posthog import tagfrom rich.console import Consolefrom pipelex import log, pretty_print_mdfrom pipelex.builder.builder import load_and_validate_bundlefrom pipelex.builder.builder_errors import PipelexBundleError-from pipelex.exceptions import PipeInputError, PipelineExecutionError+from pipelex.cli.error_handlers import (+ ErrorContext,+ handle_model_availability_error,+ handle_model_choice_error,+ handle_model_deck_preset_error,+ handle_validation_error,+)+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError+from pipelex.exceptions import (+ LibraryLoadingError,+ PipeInputError,+ PipelineExecutionError,+ PipeOperatorModelAvailabilityError,+ PipeOperatorModelChoiceError,+)+from pipelex.hub import get_telemetry_managerfrom pipelex.pipelex import Pipelexfrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.system.runtime import IntegrationMode@@ -111,8 +126,6 @@ def run_cmd(raise typer.Exit(1)async def run_pipeline(pipe_code: str | None = None, bundle_path: str | None = None):- # Initialize Pipelex- Pipelex.make(integration_mode=IntegrationMode.CLI)source_description: strif bundle_path:try:@@ -141,63 +154,79 @@ def run_cmd(typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)raise typer.Exit(1)- try:- # Load inputs if provided- pipeline_inputs = None- if inputs:- if inputs.startswith("{"):- pipeline_inputs = json.loads(inputs)- else:- try:- pipeline_inputs = load_json_dict_from_path(inputs)- typer.echo(f"Loaded inputs from: {inputs}")- except FileNotFoundError as file_not_found_exc:- typer.secho(f"Failed to load input file '{inputs}': file not found", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from file_not_found_exc- except JsonTypeError as json_type_error_exc:- typer.secho(f"Failed to parse input file '{inputs}': must be a valid JSON dictionary", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from json_type_error_exc-- # Execute pipeline- typer.secho(f"\n🚀 Executing {source_description}...\n", fg=typer.colors.GREEN, bold=True)-- try:- pipe_output = await execute_pipeline(- pipe_code=pipe_code,- inputs=pipeline_inputs,- )- except PipelineExecutionError as exc:- typer.secho(f"Failed to execute pipeline: {exc}", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from exc-- # Pretty print main_stuff unless disabled- if not no_pretty_print:- typer.echo("")- pretty_print_md(content=pipe_output.main_stuff.content.rendered_markdown(), title=f"Main output of '{pipe_code}'")- typer.echo("")-- # Save working memory to JSON unless disabled- if not no_output:- output_path = output or get_incremental_file_path(- base_path="results",- base_name=f"run_{pipe_code}",- extension="json",- )- working_memory_dict = pipe_output.working_memory.smart_dump()- save_as_json_to_path(object_to_save=working_memory_dict, path=output_path)- typer.secho(f"✅ Working memory saved to: {output_path}", fg=typer.colors.GREEN)+ # Load inputs if provided+ pipeline_inputs = None+ if inputs:+ if inputs.startswith("{"):+ pipeline_inputs = json.loads(inputs)+ else:+ try:+ pipeline_inputs = load_json_dict_from_path(inputs)+ typer.echo(f"Loaded inputs from: {inputs}")+ except FileNotFoundError as file_not_found_exc:+ typer.secho(f"Failed to load input file '{inputs}': file not found", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from file_not_found_exc+ except JsonTypeError as json_type_error_exc:+ typer.secho(f"Failed to parse input file '{inputs}': must be a valid JSON dictionary", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from json_type_error_exc++ # Execute pipeline+ typer.secho(f"\n🚀 Executing {source_description}...\n", fg=typer.colors.GREEN, bold=True)- typer.secho("✅ Pipeline execution completed successfully", fg=typer.colors.GREEN)-- except Exception as exc:- log.error(f"Error executing pipeline: {exc}")- console = Console(stderr=True)- console.print("\n[bold red]Failed to execute pipeline[/bold red]\n")- console.print_exception(show_locals=True)+ try:+ pipe_output = await execute_pipeline(+ pipe_code=pipe_code,+ inputs=pipeline_inputs,+ )+ except PipelineExecutionError as exc:+ typer.secho(f"Failed to execute pipeline: {exc}", fg=typer.colors.RED, err=True)raise typer.Exit(1) from exc- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())- tag(name=EventProperty.CLI_COMMAND, value=COMMAND)- asyncio.run(run_pipeline(pipe_code=pipe_code, bundle_path=bundle_path))+ # Pretty print main_stuff unless disabled+ if not no_pretty_print:+ typer.echo("")+ pretty_print_md(content=pipe_output.main_stuff.content.rendered_markdown(), title=f"Main output of '{pipe_code}'")+ typer.echo("")++ # Save working memory to JSON unless disabled+ if not no_output:+ output_path = output or get_incremental_file_path(+ base_path="results",+ base_name=f"run_{pipe_code}",+ extension="json",+ )+ working_memory_dict = pipe_output.working_memory.smart_dump()+ save_as_json_to_path(object_to_save=working_memory_dict, path=output_path)+ typer.secho(f"✅ Working memory saved to: {output_path}", fg=typer.colors.GREEN)++ typer.secho("✅ Pipeline execution completed successfully", fg=typer.colors.GREEN)++ # Initialize Pipelex BEFORE telemetry context to ensure proper setup+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_PIPE_RUN)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_PIPE_RUN)++ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())+ tag(name=EventProperty.CLI_COMMAND, value=COMMAND)+ asyncio.run(run_pipeline(pipe_code=pipe_code, bundle_path=bundle_path))++ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.PIPE_RUN)++ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.PIPE_RUN)++ except Exception as exc:+ log.error(f"Error executing pipeline: {exc}")+ console = Console(stderr=True)+ console.print("\n[bold red]Failed to execute pipeline[/bold red]\n")+ console.print_exception(show_locals=True)+ raise typer.Exit(1) from exc+ finally:+ pipelex_instance.teardown()diff --git a/pipelex/cli/commands/show_cmd.py b/pipelex/cli/commands/show_cmd.pyindex 931f50a85..63a45bb2c 100644--- a/pipelex/cli/commands/show_cmd.py+++ b/pipelex/cli/commands/show_cmd.py@@ -4,16 +4,23 @@ import asynciofrom typing import TYPE_CHECKING, Annotated, castimport typer-from posthog import new_context, tag+from posthog import tagfrom rich import boxfrom rich.console import Consolefrom rich.table import Tablefrom pipelex import pretty_print+from pipelex.cli.error_handlers import (+ ErrorContext,+ handle_model_deck_preset_error,+ handle_validation_error,+)+from pipelex.cogt.exceptions import ModelDeckPresetValidatonErrorfrom pipelex.cogt.model_backends.backend_library import InferenceBackendLibraryfrom pipelex.cogt.model_backends.model_lists import ModelLister-from pipelex.exceptions import PipelexCLIError, PipelexConfigError-from pipelex.hub import get_models_manager, get_pipe_library, get_required_pipe, get_telemetry_manager+from pipelex.config import ConfigPaths+from pipelex.exceptions import LibraryLoadingError, PipelexCLIError, PipelexConfigError+from pipelex.hub import get_models_manager, get_pipe_library, get_required_pipe, get_secrets_provider, get_telemetry_managerfrom pipelex.pipelex import Pipelexfrom pipelex.system.configuration.config_loader import config_managerfrom pipelex.system.runtime import IntegrationMode@@ -60,16 +67,22 @@ def do_show_pipe(pipe_code: str) -> None:def do_show_backends(show_all: bool = False) -> None:"""Display all backends and the active routing profile."""try:+ secrets_provider = get_secrets_provider()models_manager = cast("ModelManager", get_models_manager())# Load backends with or without disabled ones based on show_all flagif show_all:backend_library = InferenceBackendLibrary()- backend_library.load(include_disabled=True)+ backend_library.load(+ secrets_provider=secrets_provider,+ backends_library_path=ConfigPaths.BACKENDS_FILE_PATH,+ backends_dir_path=ConfigPaths.BACKENDS_DIR_PATH,+ include_disabled=True,+ )else:backend_library = models_manager.inference_backend_library- routing_profile_library = models_manager.routing_profile_library+ routing_profile = models_manager.routing_profileexcept Exception as exc:msg = f"Error accessing backend or routing configuration: {exc}"raise PipelexCLIError(msg) from exc@@ -115,17 +128,15 @@ def do_show_backends(show_all: bool = False) -> None:# Display routing profile informationtry:- active_profile = routing_profile_library.active_profile-- console.print(f"[bold cyan]Active Routing Profile:[/bold cyan] [green]{active_profile.name}[/green]")- if active_profile.description:- console.print(f"[dim]{active_profile.description}[/dim]")+ console.print(f"[bold cyan]Active Routing Profile:[/bold cyan] [green]{routing_profile.name}[/green]")+ if routing_profile.description:+ console.print(f"[dim]{routing_profile.description}[/dim]")- if active_profile.default:- console.print(f"[bold]Default Backend:[/bold] [cyan]{active_profile.default}[/cyan]")+ if routing_profile.default:+ console.print(f"[bold]Default Backend:[/bold] [cyan]{routing_profile.default}[/cyan]")# Display routing rules- if active_profile.routes:+ if routing_profile.routes:console.print("\n[bold]Routing Rules:[/bold]")routes_table = Table(show_header=True,@@ -137,7 +148,7 @@ def do_show_backends(show_all: bool = False) -> None:routes_table.add_column("→", style="dim", justify="center")routes_table.add_column("Target Backend", style="cyan")- for pattern, target_backend in sorted(active_profile.routes.items()):+ for pattern, target_backend in sorted(routing_profile.routes.items()):routes_table.add_row(pattern, "→", target_backend)console.print(routes_table)@@ -180,9 +191,14 @@ def list_pipes_cmd() -> None:This includes pipes from your project's .plx files and anypipes from imported packages."""- Pipelex.make(integration_mode=IntegrationMode.CLI)+ try:+ Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPES)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPES)- with new_context():+ with get_telemetry_manager().telemetry_context():tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPES}")@@ -200,9 +216,14 @@ def show_pipe_cmd(Example:pipelex show pipe hello_world"""- Pipelex.make(integration_mode=IntegrationMode.CLI)+ try:+ Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPE)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_PIPE)- with new_context():+ with get_telemetry_manager().telemetry_context():tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_PIPE}")@@ -227,18 +248,27 @@ def show_models_cmd(pipelex show models openaipipelex show models anthropic --flat"""- Pipelex.make(integration_mode=IntegrationMode.CLI)- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_MODELS}")+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_MODELS)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_MODELS)- asyncio.run(- ModelLister.list_models(- backend_name=backend_name,- flat=flat,+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_MODELS}")++ asyncio.run(+ ModelLister.list_models(+ backend_name=backend_name,+ flat=flat,+ ))- )+ finally:+ pipelex_instance.teardown()@show_app.command("backends", help="Display backend configurations and active routing profile")@@ -253,8 +283,14 @@ def show_backends_cmd(pipelex show backendspipelex show backends --all"""- Pipelex.make(integration_mode=IntegrationMode.CLI)- with new_context():+ try:+ Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_BACKENDS)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_SHOW_BACKENDS)++ with get_telemetry_manager().telemetry_context():tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_BACKENDS}")diff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.pyindex 756635d7f..92c4d4b6e 100644--- a/pipelex/cli/commands/validate_cmd.py+++ b/pipelex/cli/commands/validate_cmd.py@@ -6,16 +6,23 @@ from typing import TYPE_CHECKING, Annotatedimport clickimport typer-from posthog import new_context, tag+from posthog import tagfrom rich.console import Console-from rich.syntax import Syntaxfrom rich.traceback import Traceback-from pipelex import log, pretty_print+from pipelex import logfrom pipelex.builder.builder_errors import PipelexBundleErrorfrom pipelex.builder.builder_validation import validate_dry_run_bundle_blueprint+from pipelex.cli.error_handlers import (+ ErrorContext,+ handle_model_availability_error,+ handle_model_choice_error,+ handle_model_deck_preset_error,+ handle_validation_error,+)+from pipelex.cogt.exceptions import ModelDeckPresetValidatonErrorfrom pipelex.core.interpreter import PipelexInterpreter-from pipelex.exceptions import LibraryLoadingError, PipeInputError+from pipelex.exceptions import LibraryLoadingError, PipeInputError, PipeOperatorModelAvailabilityError, PipeOperatorModelChoiceErrorfrom pipelex.hub import get_library_manager, get_pipes, get_required_pipe, get_telemetry_managerfrom pipelex.pipe_run.dry_run import dry_run_pipe, dry_run_pipesfrom pipelex.pipelex import Pipelex@@ -25,7 +32,6 @@ from pipelex.tools.misc.package_utils import get_package_versionif TYPE_CHECKING:from pipelex.core.pipes.pipe_abstract import PipeAbstract- from pipelex.core.validation_errors import ValidationErrorDetailsProtocolconsole = Console()@@ -34,17 +40,30 @@ COMMAND = "validate"def do_validate_all_libraries_and_dry_run() -> None:"""Validate libraries and dry-run all pipes."""- pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} all")+ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except LibraryLoadingError as library_loading_error:+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION)- pipelex_instance.validate_libraries()- pipes = get_pipes()- get_telemetry_manager().track_event(EventName.PIPE_DRY_RUN, properties={EventProperty.NB_PIPES: len(pipes)})- asyncio.run(dry_run_pipes(pipes=pipes, raise_on_failure=True))- log.info("Setup sequence passed OK, config and pipelines are validated.")+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} all")++ pipelex_instance.validate_libraries()+ pipes = get_pipes()+ get_telemetry_manager().track_event(EventName.PIPE_DRY_RUN, properties={EventProperty.NB_PIPES: len(pipes)})+ asyncio.run(dry_run_pipes(pipes=pipes, raise_on_failure=True))+ log.info("Setup sequence passed OK, config and pipelines are validated.")+ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.VALIDATION)+ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.VALIDATION)+ finally:+ pipelex_instance.teardown()def validate_cmd(@@ -152,9 +171,7 @@ def validate_cmd(raise typer.Exit(1) from excexcept PipelexBundleError as bundle_error:console.print(Traceback())- typer.secho(f"\n❌ Failed to validate bundle '{bundle_path}':", fg=typer.colors.RED, err=True)- present_validation_error(details_provider=bundle_error)- raise typer.Exit(1) from bundle_error+ handle_validation_error(exc=bundle_error, context=ErrorContext.VALIDATION)except PipeInputError as exc:console.print(Traceback())typer.secho(f"\n❌ Failed to validate bundle '{bundle_path}':", fg=typer.colors.RED, err=True)@@ -176,58 +193,27 @@ def validate_cmd(raise typer.Exit(1)# Initialize Pipelex+ pipelex_instance: Pipelextry:pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)except LibraryLoadingError as library_loading_error:- typer.secho(f"Failed to validate: {library_loading_error}", fg=typer.colors.RED, err=True)- present_validation_error(details_provider=library_loading_error)- raise typer.Exit(1) from library_loading_error+ handle_validation_error(exc=library_loading_error, context=ErrorContext.VALIDATION)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION)- with new_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())- if bundle_path:- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} bundle")- else:- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} pipe")-- asyncio.run(validate_pipe(pipe_code=pipe_code, bundle_path=bundle_path))---def present_validation_error(details_provider: ValidationErrorDetailsProtocol):- console.print(details_provider)- concept_definition_errors = details_provider.get_concept_definition_errors()- if not concept_definition_errors:- return- for concept_definition_error in concept_definition_errors:- syntax_error_data = concept_definition_error.structure_class_syntax_error_data- if not syntax_error_data:- continue- message = concept_definition_error.message- code = concept_definition_error.structure_class_python_code or ""- highlight_lines: set[int] | None = None- if syntax_error_data.lineno:- highlight_lines = {syntax_error_data.lineno}- syntax = Syntax(- code=code,- lexer="python",- line_numbers=True,- word_wrap=False,- # theme="monokai", # pick any theme rich knows; omit to use default- theme="ansi_dark", # pick any theme rich knows; omit to use default- line_range=None,- highlight_lines=highlight_lines,- )- pretty_range = ""- if syntax_error_data.lineno and syntax_error_data.end_lineno:- pretty_range = f"lines {syntax_error_data.lineno} to {syntax_error_data.end_lineno}"- elif syntax_error_data.lineno:- pretty_range = f"line {syntax_error_data.lineno}"- if syntax_error_data.offset and syntax_error_data.end_offset:- pretty_range += f", column {syntax_error_data.offset} to {syntax_error_data.end_offset}"- elif syntax_error_data.offset:- pretty_range += f", column {syntax_error_data.offset}"- console.print(message)- if pretty_range:- pretty_print(f"Generated code error at {pretty_range}")- console.print(syntax)+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())+ if bundle_path:+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} bundle")+ else:+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} pipe")++ asyncio.run(validate_pipe(pipe_code=pipe_code, bundle_path=bundle_path))+ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.VALIDATION)+ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.VALIDATION)+ finally:+ pipelex_instance.teardown()diff --git a/pipelex/cli/error_handlers.py b/pipelex/cli/error_handlers.pynew file mode 100644index 000000000..377754865--- /dev/null+++ b/pipelex/cli/error_handlers.py@@ -0,0 +1,181 @@+"""Helper functions for handling and displaying CLI errors."""++from __future__ import annotations++from enum import StrEnum+from typing import TYPE_CHECKING, NoReturn++import typer+from rich.console import Console+from rich.syntax import Syntax++from pipelex.urls import URLs++if TYPE_CHECKING:+ from pipelex.cogt.exceptions import ModelDeckPresetValidatonError+ from pipelex.core.validation_errors import PipelexValidationExceptionAbstract+ from pipelex.exceptions import PipeOperatorModelAvailabilityError, PipeOperatorModelChoiceError+++class ErrorContext(StrEnum):+ """Context for error messages in CLI commands."""++ PIPE_RUN = "Pipe run"+ VALIDATION = "Pipe validation"+ BUILD = "Pipe build"++ # Pre-validation contexts (for Pipelex.make() errors)+ VALIDATION_BEFORE_SHOW_PIPES = "Pre-validation (show pipes)"+ VALIDATION_BEFORE_SHOW_PIPE = "Pre-validation (show pipe)"+ VALIDATION_BEFORE_SHOW_MODELS = "Pre-validation (show models)"+ VALIDATION_BEFORE_SHOW_BACKENDS = "Pre-validation (show backends)"+ VALIDATION_BEFORE_PIPE_RUN = "Pre-validation (pipe run)"+ VALIDATION_BEFORE_BUILD_PIPE = "Pre-validation (build pipe)"+ VALIDATION_BEFORE_BUILD_RUNNER = "Pre-validation (build runner)"+ VALIDATION_BEFORE_BUILD_INPUTS = "Pre-validation (build inputs)"+ VALIDATION_BEFORE_BUILD_ONE_SHOT = "Pre-validation (build one-shot)"+ VALIDATION_BEFORE_BUILD_PARTIAL = "Pre-validation (build partial)"+++def handle_model_choice_error(exc: PipeOperatorModelChoiceError, context: ErrorContext) -> NoReturn:+ """Handle and display PipeOperatorModelChoiceError with formatted output.++ Args:+ exc: The model choice error exception+ context: Context for the error message+ """+ console = Console(stderr=True)+ console.print(f"\n[bold red]❌ {context} failed because of a model choice could not be interpreted correctly[/bold red]\n")+ console.print(f"[bold cyan]Pipe:[/bold cyan] [yellow]'{exc.pipe_code}'[/yellow] [dim]({exc.pipe_type})[/dim]")+ console.print(f"[bold cyan]Model Type:[/bold cyan] [yellow]'{exc.model_type}'[/yellow]")+ console.print(f"[bold cyan]Model Choice:[/bold cyan] [yellow]'{exc.model_choice}'[/yellow]")+ console.print(f"\n[bold red]Error:[/bold red] {exc.message}\n")+ console.print(+ f"[bold green]💡 Tip:[/bold green] Check your model configuration in [cyan].pipelex/inference/[/cyan] "+ f"or specify a different model in the [yellow]'{exc.pipe_code}'[/yellow] pipe."+ )+ console.print(f"[dim]Learn more about the inference backend system: {URLs.backend_provider_docs}[/dim]")+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")+ raise typer.Exit(1) from exc+++def handle_model_availability_error(exc: PipeOperatorModelAvailabilityError, context: ErrorContext) -> NoReturn:+ """Handle and display PipeOperatorModelAvailabilityError with formatted output.++ Args:+ exc: The model availability error exception+ context: Context for the error message+ """+ console = Console(stderr=True)+ console.print(f"\n[bold red]❌ {context} failed because a model wasn't available[/bold red]\n")+ console.print(f"[bold cyan]Pipe:[/bold cyan] [yellow]'{exc.pipe_code}'[/yellow] [dim]({exc.pipe_type})[/dim]")+ console.print(f"[bold cyan]Model:[/bold cyan] [yellow]'{exc.model_handle}'[/yellow]")+ if exc.fallback_list:+ fallbacks_str = ", ".join([f"[yellow]{fb}[/yellow]" for fb in exc.fallback_list])+ console.print(f"[bold cyan]Fallbacks:[/bold cyan] {fallbacks_str}")+ if len(exc.pipe_stack) > 1:+ stack_str = " [dim]→[/dim] ".join([f"[yellow]{p}[/yellow]" for p in exc.pipe_stack])+ console.print(f"[bold cyan]Pipe Stack:[/bold cyan] {stack_str}")+ console.print(f"\n[bold red]Error:[/bold red] {exc}\n")+ console.print(+ f"[bold green]💡 Tip:[/bold green] Check your model configuration in [cyan].pipelex/inference/[/cyan] "+ f"or specify a different model in the [yellow]'{exc.pipe_code}'[/yellow] pipe."+ )+ console.print(f"[dim]Learn more about the inference backend system: {URLs.backend_provider_docs}[/dim]")+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")+ raise typer.Exit(1) from exc+++def handle_model_deck_preset_error(exc: ModelDeckPresetValidatonError, context: ErrorContext) -> NoReturn:+ """Handle and display ModelDeckPresetValidatonError with formatted output.++ Args:+ exc: The model deck preset validation error exception+ context: Context for the error message+ """+ console = Console(stderr=True)+ console.print(f"\n[bold red]❌ {context} failed due to model deck preset validation error[/bold red]\n")+ console.print(f"[bold cyan]Preset ID:[/bold cyan] [yellow]'{exc.preset_id}'[/yellow]")+ console.print(f"[bold cyan]Model Type:[/bold cyan] [yellow]'{exc.model_type}'[/yellow]")+ console.print(f"[bold cyan]Model Handle:[/bold cyan] [yellow]'{exc.model_handle}'[/yellow]")+ if exc.enabled_backends:+ backends_str = ", ".join([f"[yellow]{b}[/yellow]" for b in sorted(exc.enabled_backends)])+ console.print(f"[bold cyan]Enabled Backends:[/bold cyan] {backends_str}")+ console.print(f"\n[bold red]Error:[/bold red] {exc.message}\n")+ backends_str = ", ".join([f"[yellow]{b}[/yellow]" for b in sorted(exc.enabled_backends)])+ console.print(+ f"[bold green]💡 Tip:[/bold green] The preset [yellow]'{exc.preset_id}'[/yellow] references model handle "+ f"[yellow]'{exc.model_handle}'[/yellow] which is not available in any enabled backend.\n"+ f"The enabled backends are: {backends_str}."+ )+ console.print(+ "[bold]Possible solutions:[/bold]\n"+ " 1. Update the preset to use a different model\n"+ f" 2. Configure model '{exc.model_handle}' in one of your enabled backends\n"+ f" 3. Enable a backend that supports [yellow]'{exc.model_handle}'[/yellow]"+ )+ console.print(f"\n[dim]Learn more about the inference backend system: {URLs.backend_provider_docs}[/dim]")+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")+ raise typer.Exit(1) from exc+++def handle_validation_error(exc: PipelexValidationExceptionAbstract, context: ErrorContext) -> NoReturn:+ """Handle and display validation errors with formatted output.++ Args:+ exc: The validation error exception (implements ValidationErrorDetailsProtocol)+ context: Context for the error message+ """+ console = Console(stderr=True)+ console.print(f"\n[bold red]❌ {context} failed due to validation error[/bold red]\n")+ console.print(f"[bold red]Error:[/bold red] {exc}\n")++ # Display concept definition errors with syntax highlighting+ concept_definition_errors = exc.get_concept_definition_errors()+ if concept_definition_errors:+ console.print("[bold cyan]Concept Definition Errors:[/bold cyan]\n")+ for concept_definition_error in concept_definition_errors:+ syntax_error_data = concept_definition_error.structure_class_syntax_error_data+ if not syntax_error_data:+ continue++ message = concept_definition_error.message+ code = concept_definition_error.structure_class_python_code or ""+ highlight_lines: set[int] | None = None+ if syntax_error_data.lineno:+ highlight_lines = {syntax_error_data.lineno}++ syntax = Syntax(+ code=code,+ lexer="python",+ line_numbers=True,+ word_wrap=False,+ theme="ansi_dark",+ line_range=None,+ highlight_lines=highlight_lines,+ )++ # Build pretty error location+ pretty_range = ""+ if syntax_error_data.lineno and syntax_error_data.end_lineno:+ pretty_range = f"lines {syntax_error_data.lineno} to {syntax_error_data.end_lineno}"+ elif syntax_error_data.lineno:+ pretty_range = f"line {syntax_error_data.lineno}"+ if syntax_error_data.offset and syntax_error_data.end_offset:+ pretty_range += f", column {syntax_error_data.offset} to {syntax_error_data.end_offset}"+ elif syntax_error_data.offset:+ pretty_range += f", column {syntax_error_data.offset}"++ console.print(f"[yellow]{message}[/yellow]")+ if pretty_range:+ console.print(f"[dim]Generated code error at {pretty_range}[/dim]")+ console.print(syntax)+ console.print()++ console.print(+ "[bold green]💡 Tip:[/bold green] Review the error message above and check your pipeline configuration. "+ "Make sure all required fields are present and correctly formatted."+ )+ console.print(f"[dim]Learn more: {URLs.documentation}[/dim]")+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")+ raise typer.Exit(1) from excdiff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.pyindex c0a4229d4..a1d265434 100644--- a/pipelex/cogt/config_cogt.py+++ b/pipelex/cogt/config_cogt.py@@ -1,9 +1,9 @@from pipelex.cogt.exceptions import LLMConfigErrorfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobParamsDefaultsfrom pipelex.cogt.llm.llm_job_components import LLMJobConfig+from pipelex.cogt.models.model_deck_config import ModelDeckConfigfrom pipelex.plugins.fal.fal_config import FalConfigfrom pipelex.system.configuration.config_model import ConfigModel-from pipelex.tools.misc.file_utils import find_files_in_dirclass ExtractConfig(ConfigModel):@@ -44,43 +44,8 @@ class LLMConfig(ConfigModel):return template-class InferenceManagerConfig(ConfigModel):- is_auto_setup_preset_llm: bool- is_auto_setup_preset_img_gen: bool- is_auto_setup_preset_extract: bool---class InferenceConfig(ConfigModel):- inference_config_path: str-- @property- def routing_profile_library_path(self) -> str:- return f"{self.inference_config_path}/routing_profiles.toml"-- @property- def backends_library_path(self) -> str:- return f"{self.inference_config_path}/backends.toml"-- def model_specs_path(self, backend_name: str) -> str:- return f"{self.inference_config_path}/backends/{backend_name}.toml"-- def get_model_deck_paths(self) -> list[str]:- """Get all Model deck TOML file paths sorted alphabetically."""- model_deck_paths = [- str(path)- for path in find_files_in_dir(- dir_path=f"{self.inference_config_path}/deck",- pattern="*.toml",- is_recursive=True,- )- ]- model_deck_paths.sort()- return model_deck_paths--class Cogt(ConfigModel):- inference_config: InferenceConfig- inference_manager_config: InferenceManagerConfig+ model_deck_config: ModelDeckConfigllm_config: LLMConfigimg_gen_config: ImgGenConfigextract_config: ExtractConfigdiff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex 5deb982ff..ac170c6e4 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -1,3 +1,7 @@+from pipelex.cogt.extract.extract_setting import ExtractModelChoice+from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice+from pipelex.cogt.llm.llm_setting import LLMModelChoice+from pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.system.exceptions import FatalError, RootExceptionfrom pipelex.types import StrEnum@@ -38,16 +42,11 @@ class LLMWorkerError(CogtError):pass-class LLMChoiceNotFoundError(CogtError):- pass---class ExtractChoiceNotFoundError(CogtError):- pass---class ImgGenChoiceNotFoundError(CogtError):- pass+class ModelChoiceNotFoundError(CogtError):+ def __init__(self, message: str, model_type: ModelType, model_choice: LLMModelChoice | ExtractModelChoice | ImgGenModelChoice):+ self.model_type = model_type+ self.model_choice = model_choice+ super().__init__(message=message)class LLMSettingsValidationError(CogtError):@@ -62,12 +61,55 @@ class ModelDeckValidatonError(CogtError):pass+class ModelDeckPresetValidatonError(ModelDeckValidatonError):+ def __init__(+ self,+ message: str,+ model_type: ModelType,+ preset_id: str,+ model_handle: str,+ enabled_backends: set[str] | None = None,+ ):+ self.model_type = model_type+ self.preset_id = preset_id+ self.model_handle = model_handle+ self.enabled_backends = enabled_backends or set()+ super().__init__(message)++class ModelNotFoundError(CogtError):- pass+ def __init__(self, message: str, model_handle: str):+ self.model_handle = model_handle+ super().__init__(message)+++class ModelWaterfallError(ModelNotFoundError):+ def __init__(self, message: str, model_handle: str, fallback_list: list[str]):+ self.model_handle = model_handle+ self.fallback_list = fallback_list+ super().__init__(message=message, model_handle=model_handle)class LLMHandleNotFoundError(CogtError):- pass+ def __init__(self, message: str, preset_id: str, model_handle: str, enabled_backends: set[str] | None = None):+ self.preset_id = preset_id+ self.model_handle = model_handle+ self.enabled_backends = enabled_backends or set()+ super().__init__(message)+++class ImgGenHandleNotFoundError(CogtError):+ def __init__(self, message: str, preset_id: str, model_handle: str):+ self.preset_id = preset_id+ self.model_handle = model_handle+ super().__init__(message)+++class ExtractHandleNotFoundError(CogtError):+ def __init__(self, message: str, preset_id: str, model_handle: str):+ self.preset_id = preset_id+ self.model_handle = model_handle+ super().__init__(message)class LLMModelPlatformError(ValueError, CogtError):@@ -163,7 +205,7 @@ class RoutingProfileLibraryNotFoundError(CogtError):pass-class RoutingProfileValidationError(CogtError):+class RoutingProfileBlueprintValueError(CogtError, ValueError):pass@@ -211,7 +253,7 @@ class InferenceBackendLibraryError(CogtError):pass-class RoutingProfileError(CogtError):+class RoutingProfileDisabledBackendError(CogtError):passdiff --git a/pipelex/cogt/img_gen/img_gen_setting.py b/pipelex/cogt/img_gen/img_gen_setting.pyindex e2708914a..124f31ad0 100644--- a/pipelex/cogt/img_gen/img_gen_setting.py+++ b/pipelex/cogt/img_gen/img_gen_setting.py@@ -2,7 +2,6 @@ from typing import Unionfrom pydantic import Field, model_validator-from pipelex.cogt.exceptions import ImgGenSettingsValidationErrorfrom pipelex.cogt.img_gen.img_gen_job_components import Qualityfrom pipelex.system.configuration.config_model import ConfigModelfrom pipelex.types import Self@@ -20,7 +19,7 @@ class ImgGenSetting(ConfigModel):def validate_quality_or_nb_steps(self) -> Self:if self.quality is not None and self.nb_steps is not None:msg = "ImgGenSetting cannot have both 'quality' and 'nb_steps' specified. Use one or the other."- raise ImgGenSettingsValidationError(msg)+ raise ValueError(msg)return selfdef desc(self) -> str:diff --git a/pipelex/cogt/inference/inference_manager.py b/pipelex/cogt/inference/inference_manager.pyindex c2cf6a9c0..38b405b74 100644--- a/pipelex/cogt/inference/inference_manager.py+++ b/pipelex/cogt/inference/inference_manager.py@@ -1,7 +1,6 @@from typing_extensions import overridefrom pipelex import log-from pipelex.cogt.exceptions import InferenceManagerWorkerSetupErrorfrom pipelex.cogt.extract.extract_worker_abstract import ExtractWorkerAbstractfrom pipelex.cogt.extract.extract_worker_factory import ExtractWorkerFactoryfrom pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract@@ -10,8 +9,6 @@ from pipelex.cogt.inference.inference_manager_protocol import InferenceManagerPrfrom pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstractfrom pipelex.cogt.llm.llm_worker_factory import LLMWorkerFactoryfrom pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract-from pipelex.cogt.model_backends.model_spec import InferenceModelSpec-from pipelex.config import get_configfrom pipelex.hub import get_models_manager, get_report_delegate@@ -59,9 +56,9 @@ class InferenceManager(InferenceManagerProtocol):def _setup_one_internal_llm_worker(self,- inference_model: InferenceModelSpec,llm_handle: str,) -> LLMWorkerInternalAbstract:+ inference_model = get_models_manager().get_inference_model(model_handle=llm_handle)llm_worker = LLMWorkerFactory.make_llm_worker(inference_model=inference_model,reporting_delegate=get_report_delegate(),@@ -71,17 +68,10 @@ class InferenceManager(InferenceManagerProtocol):@overridedef get_llm_worker(self, llm_handle: str) -> LLMWorkerAbstract:- if llm_worker := self.llm_workers.get(llm_handle):- return llm_worker- if not get_config().cogt.inference_manager_config.is_auto_setup_preset_llm:- msg = f"No LLM worker for '{llm_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_llm"- raise InferenceManagerWorkerSetupError(msg)-- inference_model = get_models_manager().get_inference_model(model_handle=llm_handle)- return self._setup_one_internal_llm_worker(- inference_model=inference_model,- llm_handle=llm_handle,- )+ llm_worker = self.llm_workers.get(llm_handle)+ if llm_worker is None:+ llm_worker = self._setup_one_internal_llm_worker(llm_handle=llm_handle)+ return llm_worker@overridedef set_llm_worker_from_external_plugin(@@ -112,10 +102,6 @@ class InferenceManager(InferenceManagerProtocol):def get_img_gen_worker(self, img_gen_handle: str) -> ImgGenWorkerAbstract:img_gen_worker = self.img_gen_workers.get(img_gen_handle)if img_gen_worker is None:- if not get_config().cogt.inference_manager_config.is_auto_setup_preset_img_gen:- msg = f"Found no ImgGen worker for '{img_gen_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_img_gen"- raise InferenceManagerWorkerSetupError(msg)-img_gen_worker = self._setup_one_img_gen_worker(img_gen_handle=img_gen_handle)return img_gen_worker@@ -125,9 +111,9 @@ class InferenceManager(InferenceManagerProtocol):def _setup_one_extract_worker(self,- inference_model: InferenceModelSpec,extract_handle: str,) -> ExtractWorkerAbstract:+ inference_model = get_models_manager().get_inference_model(model_handle=extract_handle)extract_worker = self.extract_worker_factory.make_extract_worker(inference_model=inference_model,reporting_delegate=get_report_delegate(),@@ -137,14 +123,7 @@ class InferenceManager(InferenceManagerProtocol):@overridedef get_extract_worker(self, extract_handle: str) -> ExtractWorkerAbstract:- if extract_worker := self.extract_workers.get(extract_handle):- return extract_worker- if not get_config().cogt.inference_manager_config.is_auto_setup_preset_extract:- msg = f"Found no Extract worker for '{extract_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_extract"- raise InferenceManagerWorkerSetupError(msg)-- inference_model = get_models_manager().get_inference_model(model_handle=extract_handle)- return self._setup_one_extract_worker(- inference_model=inference_model,- extract_handle=extract_handle,- )+ extract_worker = self.extract_workers.get(extract_handle)+ if extract_worker is None:+ extract_worker = self._setup_one_extract_worker(extract_handle=extract_handle)+ return extract_workerdiff --git a/pipelex/cogt/llm/llm_setting.py b/pipelex/cogt/llm/llm_setting.pyindex d40d0d364..5e5933b41 100644--- a/pipelex/cogt/llm/llm_setting.py+++ b/pipelex/cogt/llm/llm_setting.py@@ -48,7 +48,7 @@ class LLMSettingChoices(ConfigModel):for_text: LLMModelChoice | Nonefor_object: LLMModelChoice | None- def list_choices(self) -> set[str]:+ def list_choice_strings(self) -> set[str]:return {c for c in (self.for_text, self.for_object) if isinstance(c, str)}@classmethoddiff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.pyindex 9e956e421..0317e9702 100644--- a/pipelex/cogt/model_backends/backend_library.py+++ b/pipelex/cogt/model_backends/backend_library.py@@ -1,3 +1,4 @@+from functools import partialfrom typing import TYPE_CHECKING, Any, castfrom pydantic import Field, RootModel, ValidationError@@ -11,16 +12,31 @@ from pipelex.cogt.exceptions import (InferenceModelSpecError,)from pipelex.cogt.model_backends.backend import InferenceBackend-from pipelex.cogt.model_backends.backend_factory import InferenceBackendBlueprint, InferenceBackendFactory-from pipelex.cogt.model_backends.model_spec_factory import InferenceModelSpecBlueprint, InferenceModelSpecFactory-from pipelex.config import get_config+from pipelex.cogt.model_backends.backend_factory import (+ InferenceBackendBlueprint,+ InferenceBackendFactory,+)+from pipelex.cogt.model_backends.model_spec_factory import (+ InferenceModelSpecBlueprint,+ InferenceModelSpecFactory,+)from pipelex.system.configuration.config_model import ConfigModelfrom pipelex.system.environment import get_optional_envfrom pipelex.system.runtime import runtime_manager-from pipelex.tools.misc.dict_utils import apply_to_strings_recursive, extract_vars_from_strings_recursive+from pipelex.tools.misc.dict_utils import (+ apply_to_strings_recursive,+ extract_vars_from_strings_recursive,+)from pipelex.tools.misc.placeholder import value_is_placeholderfrom pipelex.tools.misc.toml_utils import load_toml_from_path-from pipelex.tools.secrets.secrets_utils import UnknownVarPrefixError, VarFallbackPatternError, VarNotFoundError, substitute_vars+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract+from pipelex.tools.secrets.secrets_utils import (+ UnknownVarPrefixError,+ VarFallbackPatternError,+ VarNotFoundError,+ substitute_vars,+)+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorfrom pipelex.types import Selfif TYPE_CHECKING:@@ -64,16 +80,20 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):def make_empty(cls) -> Self:return cls(root={})- def load(self, include_disabled: bool = False):- backends_library_path = get_config().cogt.inference_config.backends_library_path+ def load(self, secrets_provider: SecretsProviderAbstract, backends_library_path: str, backends_dir_path: str, include_disabled: bool = False):try:backends_dict = load_toml_from_path(path=backends_library_path)except FileNotFoundError as file_not_found_exc:msg = f"Could not find inference backend library at '{backends_library_path}': {file_not_found_exc}"raise InferenceBackendLibraryNotFoundError(msg) from file_not_found_excexcept ValidationError as exc:- msg = f"Invalid inference backend library configuration in '{backends_library_path}': {exc}"+ valiation_error_msg = format_pydantic_validation_error(exc)+ msg = f"Invalid inference backend library configuration in '{backends_library_path}': {valiation_error_msg}"raise InferenceBackendLibraryValidationError(msg) from exc++ # Create a partial function with the secrets provider bound+ substitute_vars_with_provider = partial(substitute_vars, secrets_provider=secrets_provider)+for backend_name, backend_dict in backends_dict.items():# We'll split the read settings into standard fields and extra configstandard_fields = InferenceBackendBlueprint.model_fields.keys()@@ -85,7 +105,7 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):if runtime_manager.is_ci_testing and backend_name == "vertexai":continuetry:- inference_backend_blueprint_dict = apply_to_strings_recursive(inference_backend_blueprint_dict_raw, substitute_vars)+ inference_backend_blueprint_dict = apply_to_strings_recursive(inference_backend_blueprint_dict_raw, substitute_vars_with_provider)except VarFallbackPatternError as var_fallback_pattern_exc:msg = f"Variable substitution failed due to a pattern error in file '{backends_library_path}':\n{var_fallback_pattern_exc}"key_name = "unknown"@@ -123,13 +143,11 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):extra_config[key] = inference_backend_blueprint_dict.pop(key)backend_blueprint = InferenceBackendBlueprint.model_validate(inference_backend_blueprint_dict)- path_to_model_specs_toml = get_config().cogt.inference_config.model_specs_path(backend_name=backend_name)+ path_to_model_specs_toml = f"{backends_dir_path}/{backend_name}.toml"try:- model_specs_dict_raw = load_toml_from_path(- path=path_to_model_specs_toml,- )+ model_specs_dict_raw = load_toml_from_path(path=path_to_model_specs_toml)try:- model_specs_dict = apply_to_strings_recursive(model_specs_dict_raw, substitute_vars)+ model_specs_dict = apply_to_strings_recursive(model_specs_dict_raw, substitute_vars_with_provider)except (VarNotFoundError, UnknownVarPrefixError) as exc:msg = f"Variable substitution failed in file '{path_to_model_specs_toml}': {exc}"raise InferenceModelSpecError(msg) from exc@@ -262,3 +280,6 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):def get_inference_backend(self, backend_name: str) -> InferenceBackend | None:return self.root.get(backend_name)++ def all_enabled_backends(self) -> list[str]:+ return [backend_name for backend_name, backend in self.root.items() if backend.enabled]diff --git a/pipelex/cogt/model_routing/routing_models.py b/pipelex/cogt/model_routing/routing_models.pyindex b289b8509..f0fe818d8 100644--- a/pipelex/cogt/model_routing/routing_models.py+++ b/pipelex/cogt/model_routing/routing_models.py@@ -15,3 +15,4 @@ class BackendMatchForModel(BaseModel):routing_profile_name: strmatching_method: BackendMatchingMethodmatched_pattern: str | None+ fallback_order: list[str] | None = Nonediff --git a/pipelex/cogt/model_routing/routing_profile.py b/pipelex/cogt/model_routing/routing_profile.pyindex 7a14f01e4..ae7d6aeef 100644--- a/pipelex/cogt/model_routing/routing_profile.py+++ b/pipelex/cogt/model_routing/routing_profile.py@@ -12,29 +12,40 @@ class RoutingProfile(ConfigModel):description: str | None = Nonedefault: str | None = Noneroutes: dict[str, str] = Field(default_factory=dict) # Pattern -> Backend mapping+ optional_routes: dict[str, str] = Field(default_factory=dict)+ fallback_order: list[str] | None = None # Ordered list of backends for fallback- def get_backend_match_for_model(self, model_name: str) -> BackendMatchForModel | None:+ def get_backend_match_for_model(self, enabled_backends: list[str], model_name: str) -> BackendMatchForModel | None:"""Get the backend name for a given model name.Args:+ enabled_backends: List of enabled backendsmodel_name: Name of the model to routeReturns:Backend name to use for this model"""+ possible_routes = self.routes+ for pattern, backend in self.optional_routes.items():+ if backend not in enabled_backends:+ continue+ possible_routes[pattern] = backend+# Check exact matches first- if model_name in self.routes:+ if (backend_name := possible_routes.get(model_name)) and (backend_name in enabled_backends):return BackendMatchForModel(model_name=model_name,- backend_name=self.routes[model_name],+ backend_name=possible_routes[model_name],routing_profile_name=self.name,matching_method=BackendMatchingMethod.EXACT_MATCH,matched_pattern=None,)# Check pattern matches- for pattern, backend in self.routes.items():+ for pattern, backend in possible_routes.items():+ if backend not in enabled_backends:+ continueif matches_wildcard_pattern(model_name, pattern):return BackendMatchForModel(model_name=model_name,@@ -44,13 +55,26 @@ class RoutingProfile(ConfigModel):matched_pattern=pattern,)- # Return default backend- if self.default:+ # Validate fallback_order if set+ validated_fallback_order: list[str] | None = None+ if self.fallback_order:+ validated_fallback_order = [backend for backend in self.fallback_order if backend in enabled_backends]++ # Determine primary backend for DEFAULT matching+ primary_backend: str | None = None+ if self.default and self.default in enabled_backends:+ primary_backend = self.default+ elif validated_fallback_order:+ primary_backend = validated_fallback_order[0]++ # Return default backend match if we have a primary backend+ if primary_backend:return BackendMatchForModel(model_name=model_name,- backend_name=self.default,+ backend_name=primary_backend,routing_profile_name=self.name,matching_method=BackendMatchingMethod.DEFAULT,matched_pattern=None,+ fallback_order=validated_fallback_order,)return Nonediff --git a/pipelex/cogt/model_routing/routing_profile_factory.py b/pipelex/cogt/model_routing/routing_profile_factory.pyindex a3c6e71e7..e9e3816f6 100644--- a/pipelex/cogt/model_routing/routing_profile_factory.py+++ b/pipelex/cogt/model_routing/routing_profile_factory.py@@ -1,7 +1,9 @@-from pydantic import Field+from pydantic import Field, model_validator+from pipelex.cogt.exceptions import RoutingProfileBlueprintValueErrorfrom pipelex.cogt.model_routing.routing_profile import RoutingProfilefrom pipelex.system.configuration.config_model import ConfigModel+from pipelex.types import Selfclass RoutingProfileBlueprint(ConfigModel):@@ -10,6 +12,16 @@ class RoutingProfileBlueprint(ConfigModel):description: strdefault: str | None = Noneroutes: dict[str, str] = Field(default_factory=dict)+ optional_routes: dict[str, str] = Field(default_factory=dict)+ fallback_order: list[str] | None = None++ @model_validator(mode="after")+ def validate_routes(self) -> Self:+ for pattern in self.optional_routes:+ if pattern in self.routes:+ msg = f"Pattern '{pattern}' is both in routes and optional_routes"+ raise RoutingProfileBlueprintValueError(msg)+ return selfclass RoutingProfileLibraryBlueprint(ConfigModel):@@ -43,4 +55,5 @@ class RoutingProfileFactory:description=blueprint.description,default=blueprint.default,routes=blueprint.routes,+ fallback_order=blueprint.fallback_order,)diff --git a/pipelex/cogt/model_routing/routing_profile_library.py b/pipelex/cogt/model_routing/routing_profile_library.pydeleted file mode 100644index 2186796a5..000000000--- a/pipelex/cogt/model_routing/routing_profile_library.py+++ /dev/null@@ -1,99 +0,0 @@-from pydantic import Field, RootModel, ValidationError--from pipelex import log-from pipelex.cogt.exceptions import RoutingProfileLibraryError, RoutingProfileLibraryNotFoundError, RoutingProfileValidationError-from pipelex.cogt.model_routing.routing_models import BackendMatchForModel-from pipelex.cogt.model_routing.routing_profile import RoutingProfile-from pipelex.cogt.model_routing.routing_profile_factory import (- RoutingProfileFactory,- RoutingProfileLibraryBlueprint,-)-from pipelex.config import get_config-from pipelex.tools.misc.toml_utils import load_toml_from_path-from pipelex.types import Self--RoutingProfileLibraryRoot = dict[str, RoutingProfile]---class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):- """Library for managing routing profile configurations."""-- root: RoutingProfileLibraryRoot = Field(default_factory=dict)- _active_config: str | None = None-- @property- def active_profile(self) -> RoutingProfile:- if not self._active_config:- msg = "No active routing profile loaded"- raise RoutingProfileLibraryError(msg)- if self._active_config not in self.root:- msg = f"Active routing profile '{self._active_config}' not found in loaded routing profile library"- raise RoutingProfileLibraryError(msg)- return self.root[self._active_config]-- @classmethod- def make_empty(cls) -> Self:- return cls(root={})-- def reset(self) -> None:- self.root = {}-- def load(self) -> None:- """Load the routing profile library configuration from TOML file."""- routing_profile_library_path = get_config().cogt.inference_config.routing_profile_library_path-- try:- catalog_dict = load_toml_from_path(path=routing_profile_library_path)- except FileNotFoundError as not_found_exc:- msg = f"Could not find routing profile library at '{routing_profile_library_path}': {not_found_exc}"- raise RoutingProfileLibraryNotFoundError(msg) from not_found_exc-- try:- catalog_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)- except ValidationError as exc:- msg = f"Invalid routing profile library configuration in '{routing_profile_library_path}': {exc}"- raise RoutingProfileValidationError(msg) from exc-- # Validate that the active config exists- if catalog_blueprint.active not in catalog_blueprint.profiles:- msg = f"Active profile '{catalog_blueprint.active}' not found in library. Available profiles: {list(catalog_blueprint.profiles.keys())}"- raise RoutingProfileLibraryError(msg)-- # Load all profiles- self.root = {}- for config_name, config_blueprint in catalog_blueprint.profiles.items():- self.root[config_name] = RoutingProfileFactory.make_routing_profile(- name=config_name,- blueprint=config_blueprint,- )- self._active_config = catalog_blueprint.active-- log.verbose(f"Loaded routing profile library with active profile: '{self._active_config}'")- log.verbose(f"Available profiles: {list(self.root.keys())}")-- def get_backend_match_for_model_from_active_routing_profile(self, model_name: str) -> BackendMatchForModel | None:- """Get the backend name for a given model.-- Args:- model_name: Name of the model to route-- Returns:- Backend name to use for this model-- Raises:- RoutingProfileLibraryError: If no active profile is set or config not found-- """- profile = self.active_profile- return profile.get_backend_match_for_model(model_name)-- def list_routing_profile_names(self) -> list[str]:- """Get a list of all available routing profile names."""- return list(self.root.keys())-- def get_required_routing_profile(self, routing_profile_name: str) -> RoutingProfile:- routing_profile = self.root.get(routing_profile_name)- if not routing_profile:- msg = f"Routing profile '{routing_profile_name}' not found in loaded routing profile library"- raise RoutingProfileLibraryError(msg)- return routing_profilediff --git a/pipelex/cogt/model_routing/routing_profile_loader.py b/pipelex/cogt/model_routing/routing_profile_loader.pynew file mode 100644index 000000000..435d45822--- /dev/null+++ b/pipelex/cogt/model_routing/routing_profile_loader.py@@ -0,0 +1,61 @@+from pydantic import ValidationError++from pipelex import log+from pipelex.cogt.exceptions import ModelManagerError, RoutingProfileLibraryError+from pipelex.cogt.model_routing.routing_profile import RoutingProfile+from pipelex.cogt.model_routing.routing_profile_factory import RoutingProfileFactory, RoutingProfileLibraryBlueprint+from pipelex.tools.misc.toml_utils import load_toml_from_path+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error+++def load_active_routing_profile(routing_profile_library_path: str, enabled_backends: list[str]) -> RoutingProfile:+ """Load the active routing profile from the routing profile library from TOML file."""+ # Load the routing profile library from TOML file+ try:+ catalog_dict = load_toml_from_path(path=routing_profile_library_path)+ except FileNotFoundError as not_found_exc:+ msg = f"Could not find routing profile library at '{routing_profile_library_path}': {not_found_exc}"+ raise ModelManagerError(msg) from not_found_exc++ # Validate the routing profile library configuration+ try:+ routing_profile_library_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)+ except ValidationError as exc:+ valiation_error_msg = format_pydantic_validation_error(exc)+ msg = f"Invalid routing profile library configuration in '{routing_profile_library_path}': {valiation_error_msg}"+ raise ModelManagerError(msg) from exc++ # Validate that the active profile exists+ profile_names = ", ".join(list(routing_profile_library_blueprint.profiles.keys()))+ active_profile_name = routing_profile_library_blueprint.active+ if active_profile_name not in profile_names:+ msg = f"Active profile '{active_profile_name}' not found in profile routing library. Available profiles: {profile_names}"+ raise ModelManagerError(msg)++ # Load all profiles+ active_profile_blueprint = routing_profile_library_blueprint.profiles[active_profile_name]+ active_profile = RoutingProfileFactory.make_routing_profile(+ name=active_profile_name,+ blueprint=active_profile_blueprint,+ )+ if active_profile.default and active_profile.default not in enabled_backends:+ msg = (+ f"Default backend '{active_profile.default}' set for routing profile '{active_profile_name}' is not enabled. "+ f"You must either enable backend '{active_profile.default}' or set a different default backend for profile '{active_profile_name}', "+ "or select a different routing profile."+ )+ raise RoutingProfileLibraryError(msg)++ # Raise error for routes that use disabled backends+ for backend_name in active_profile.routes.values():+ if backend_name not in enabled_backends:+ msg = f"Backend '{backend_name}', required for profile '{active_profile_name}' is not enabled"+ raise RoutingProfileLibraryError(msg)++ seen_disabled_backends: set[str] = set()+ for backend_name in active_profile.routes.values():+ if backend_name not in enabled_backends and backend_name not in seen_disabled_backends:+ msg = f"Backend '{backend_name}', required for profile '{active_profile_name}' is not enabled"+ log.info(msg)+ seen_disabled_backends.add(backend_name)+ return active_profilediff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.pyindex 2c9c2d650..358820cac 100644--- a/pipelex/cogt/models/model_deck.py+++ b/pipelex/cogt/models/model_deck.py@@ -1,16 +1,17 @@-from typing import Union--from pydantic import Field, field_validator, model_validator+from pydantic import Field, PrivateAttr, field_validator, model_validatorfrom pipelex import log+from pipelex.cogt.config_cogt import ModelDeckConfigfrom pipelex.cogt.exceptions import (- ExtractChoiceNotFoundError,- ImgGenChoiceNotFoundError,- LLMChoiceNotFoundError,+ ExtractHandleNotFoundError,+ ImgGenHandleNotFoundError,LLMHandleNotFoundError,LLMSettingsValidationError,+ ModelChoiceNotFoundError,+ ModelDeckPresetValidatonError,ModelDeckValidatonError,ModelNotFoundError,+ ModelWaterfallError,)from pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSettingfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSetting@@ -22,14 +23,16 @@ from pipelex.cogt.llm.llm_setting import ()from pipelex.cogt.model_backends.model_constraints import ModelConstraintsfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.system.configuration.config_model import ConfigModelfrom pipelex.system.exceptions import ConfigValidationError+from pipelex.system.runtime import ProblemReaction+from pipelex.tools.misc.toml_utils import load_toml_from_path_if_existsfrom pipelex.types import Self+from pipelex.urls import URLsLLM_PRESET_DISABLED = "disabled"-Waterfall = Union[str, list[str]]-class LLMDeckBlueprint(ConfigModel):presets: dict[str, LLMSetting] = Field(default_factory=dict)@@ -51,7 +54,8 @@ class ImgGenDeckBlueprint(ConfigModel):class ModelDeckBlueprint(ConfigModel):- aliases: dict[str, Waterfall] = Field(default_factory=dict)+ aliases: dict[str, str] = Field(default_factory=dict)+ waterfalls: dict[str, list[str]] = Field(default_factory=dict)llm: LLMDeckBlueprintextract: ExtractDeckBlueprint@@ -59,8 +63,13 @@ class ModelDeckBlueprint(ConfigModel):class ModelDeck(ConfigModel):+ model_deck_config: ModelDeckConfiginference_models: dict[str, InferenceModelSpec] = Field(default_factory=dict)- aliases: dict[str, Waterfall] = Field(default_factory=dict)+ aliases: dict[str, str] = Field(default_factory=dict)+ waterfalls: dict[str, list[str]] = Field(default_factory=dict)++ # Track which model_handle fallback warnings have been logged to avoid duplicates+ _logged_fallback_warnings: set[str] = PrivateAttr(default_factory=set[str])llm_presets: dict[str, LLMSetting] = Field(default_factory=dict)llm_choice_defaults: LLMSettingChoicesDefaults@@ -75,20 +84,28 @@ class ModelDeck(ConfigModel):img_gen_presets: dict[str, ImgGenSetting] = Field(default_factory=dict)img_gen_choice_default: ImgGenModelChoice- def check_llm_setting(+ def is_model_handle_defined(self, model_handle: str) -> bool:+ all_handles: set[str] = set()+ all_handles.update(self.inference_models.keys())+ all_handles.update(self.aliases.keys())+ if self.model_deck_config.is_model_fallback_enabled:+ all_handles.update(self.waterfalls.keys())+ return model_handle in all_handles++ def check_llm_choice(self,- llm_setting_or_preset_id: LLMModelChoice,+ llm_choice: LLMModelChoice,is_disabled_allowed: bool = False,):- if isinstance(llm_setting_or_preset_id, LLMSetting):+ if isinstance(llm_choice, LLMSetting):return- preset_id: str = llm_setting_or_preset_id+ preset_id: str = llm_choiceif preset_id in self.llm_presets:returnif preset_id == LLM_PRESET_DISABLED and is_disabled_allowed:return- msg = f"llm preset id '{preset_id}' not found in deck"- raise LLMChoiceNotFoundError(msg)+ msg = f"LLM preset '{preset_id}' was not found in the model deck"+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.LLM, model_choice=llm_choice)def get_llm_setting(self, llm_choice: LLMModelChoice) -> LLMSetting:if isinstance(llm_choice, LLMSetting):@@ -98,8 +115,8 @@ class ModelDeck(ConfigModel):return llm_presetif self.is_handle_defined(model_handle=llm_choice):return LLMSetting(model=llm_choice, temperature=0.7, max_tokens=None)- msg = f"LLM choice '{llm_choice}' not found in deck"- raise LLMChoiceNotFoundError(msg)+ msg = f"LLM choice '{llm_choice}' was not found in the model deck"+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.LLM, model_choice=llm_choice)def get_extract_setting(self, extract_choice: ExtractModelChoice) -> ExtractSetting:if isinstance(extract_choice, ExtractSetting):@@ -109,8 +126,8 @@ class ModelDeck(ConfigModel):return extract_presetif self.is_handle_defined(model_handle=extract_choice):return ExtractSetting(model=extract_choice)- msg = f"Extract choice '{extract_choice}' not found in deck"- raise ExtractChoiceNotFoundError(msg)+ msg = f"Extract choice '{extract_choice}' was not found in the model deck"+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.TEXT_EXTRACTOR, model_choice=extract_choice)def get_img_gen_setting(self, img_gen_choice: ImgGenModelChoice) -> ImgGenSetting:if isinstance(img_gen_choice, ImgGenSetting):@@ -120,8 +137,8 @@ class ModelDeck(ConfigModel):return img_gen_presetif self.is_handle_defined(model_handle=img_gen_choice):return ImgGenSetting(model=img_gen_choice)- msg = f"Image generation choice '{img_gen_choice}' not found in deck"- raise ImgGenChoiceNotFoundError(msg)+ msg = f"Image generation choice '{img_gen_choice}' was not found in the model deck"+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.IMG_GEN, model_choice=img_gen_choice)@classmethoddef final_validate(cls, deck: Self):@@ -166,61 +183,209 @@ class ModelDeck(ConfigModel):@field_validator("llm_choice_overrides", mode="after")@classmethod- def validate_llm_choice_overrides(cls, value: LLMSettingChoices) -> LLMSettingChoices:+ def validate_llm_choice_disabled_overrides(cls, value: LLMSettingChoices) -> LLMSettingChoices:if value.for_text == LLM_PRESET_DISABLED:value.for_text = Noneif value.for_object == LLM_PRESET_DISABLED:value.for_object = Nonereturn value+ @model_validator(mode="after")+ def validate_llm_choice_overrides(self) -> Self:+ for llm_preset_id in self.llm_choice_overrides.list_choice_strings():+ self.check_llm_choice(llm_choice=llm_preset_id)+ return self+def validate_llm_presets(self) -> Self:for llm_preset_id, llm_setting in self.llm_presets.items():- if llm_setting.model not in self.inference_models:- msg = f"llm_handle '{llm_setting.model}' for llm_preset '{llm_preset_id}' not found in deck"- raise LLMHandleNotFoundError(msg)+ if not self.is_model_handle_defined(model_handle=llm_setting.model):+ enabled_backends = {model.backend_name for model in self.inference_models.values()}+ msg = f"LLM handle '{llm_setting.model}' for llm preset '{llm_preset_id}' was not found in the model deck"+ raise LLMHandleNotFoundError(+ message=msg,+ preset_id=llm_preset_id,+ model_handle=llm_setting.model,+ enabled_backends=enabled_backends,+ )return self- @model_validator(mode="after")- def validate_llm_setting_overrides(self) -> Self:- self._validate_llm_choices(llm_choices=self.llm_choice_overrides)+ def validate_img_gen_presets(self) -> Self:+ for img_gen_preset_id, img_gen_setting in self.img_gen_presets.items():+ if not self.is_model_handle_defined(model_handle=img_gen_setting.model):+ msg = f"Image generation handle '{img_gen_setting.model}' for preset '{img_gen_preset_id}' was not found in the model deck"+ raise ImgGenHandleNotFoundError(+ message=msg,+ preset_id=img_gen_preset_id,+ model_handle=img_gen_setting.model,+ )+ return self++ def validate_extract_presets(self) -> Self:+ for extract_preset_id, extract_setting in self.extract_presets.items():+ if not self.is_model_handle_defined(model_handle=extract_setting.model):+ msg = f"Extract handle '{extract_setting.model}' for extract preset '{extract_preset_id}' was not found in the model deck"+ raise ExtractHandleNotFoundError(+ message=msg,+ preset_id=extract_preset_id,+ model_handle=extract_setting.model,+ )return self- def _validate_llm_choices(self, llm_choices: LLMSettingChoices):- for llm_setting in llm_choices.list_choices():- self.check_llm_setting(llm_setting_or_preset_id=llm_setting)+ def validate_registered_models(self):+ self.validate_inference_models()+ try:+ self.validate_llm_presets()+ except LLMHandleNotFoundError as exc:+ match self.model_deck_config.missing_presets_reaction:+ case ProblemReaction.RAISE:+ msg = f"Failed to validate all LLM presets: {exc}"+ raise ModelDeckPresetValidatonError(+ message=msg,+ model_type=ModelType.LLM,+ preset_id=exc.preset_id,+ model_handle=exc.model_handle,+ enabled_backends=exc.enabled_backends,+ ) from exc+ case ProblemReaction.LOG:+ log.warning(f"LLM handle not found: {exc}")+ case ProblemReaction.NONE:+ pass+ try:+ self.validate_img_gen_presets()+ except ImgGenHandleNotFoundError as exc:+ match self.model_deck_config.missing_presets_reaction:+ case ProblemReaction.RAISE:+ msg = f"Failed to validate all ImgGen presets: {exc}"+ raise ModelDeckPresetValidatonError(+ message=msg,+ model_type=ModelType.IMG_GEN,+ preset_id=exc.preset_id,+ model_handle=exc.model_handle,+ ) from exc+ case ProblemReaction.LOG:+ log.warning(f"ImgGen handle not found: {exc}")+ case ProblemReaction.NONE:+ pass+ try:+ self.validate_extract_presets()+ except ExtractHandleNotFoundError as exc:+ match self.model_deck_config.missing_presets_reaction:+ case ProblemReaction.RAISE:+ msg = f"Failed to validate all Extract presets: {exc}"+ raise ModelDeckPresetValidatonError(+ message=msg,+ model_type=ModelType.TEXT_EXTRACTOR,+ preset_id=exc.preset_id,+ model_handle=exc.model_handle,+ ) from exc+ case ProblemReaction.LOG:+ log.warning(f"Extract handle not found: {exc}")+ case ProblemReaction.NONE:+ pass++ def validate_inference_models(self):+ for model_handle in self.inference_models:+ self.get_required_inference_model(model_handle=model_handle)++ def _is_model_available_in_backend(self, model_handle: str, backend_name: str) -> bool | None:+ """Check if a model is available from a specific backend.++ This is a low-level check that reads the backend TOML file directly,+ so it works even if the backend is disabled. Best-effort: returns False+ if the file can't be read or parsed.++ Args:+ model_handle: The model handle/name to check for+ backend_name: The backend name (e.g., 'pipelex_inference')++ Returns:+ True if the model is defined in the backend's TOML file, False otherwise+ """+ backend_file_path = f".pipelex/inference/backends/{backend_name}.toml"+ try:+ backend_toml = load_toml_from_path_if_exists(backend_file_path)+ if backend_toml is None:+ return None+ # Check if model_handle exists as a top-level key (section) in the TOML+ # Exclude special sections like 'defaults'+ return model_handle in backend_toml and model_handle != "defaults"+ except Exception:+ # Best-effort: if anything goes wrong, just return None+ return Nonedef get_optional_inference_model(self, model_handle: str) -> InferenceModelSpec | None:if inference_model := self.inference_models.get(model_handle):return inference_model- if redirection := self.aliases.get(model_handle):- log.verbose(f"Redirection for '{model_handle}': {redirection}")- if isinstance(redirection, str):- alias_list = [redirection]- else:- alias_list = redirection- for alias in alias_list:- if inference_model := self.get_optional_inference_model(model_handle=alias):+ if alias := self.aliases.get(model_handle):+ log.verbose(f"Alias for '{model_handle}': {alias}")+ return self.get_optional_inference_model(model_handle=alias)+ if fallback_list := self.waterfalls.get(model_handle):+ ideal_model_handle = fallback_list[0]+ log.verbose(f"Fallback list for '{model_handle}': {fallback_list}")+ for fallback_index, fallback in enumerate(fallback_list):+ if fallback_index > 0 and not self.model_deck_config.is_model_fallback_enabled:+ # Waterfall disabled, so we raise an error+ fallback_list_str = " → ".join(fallback_list)+ msg = (+ f"Model handle '{model_handle}' is a waterfall (i.e. a list of models to try in order), which resolves to "+ f"•[ {fallback_list_str} ]•, but model fallbacks are disabled "+ f"so only the first item in the list, '{ideal_model_handle}', is acceptable but it was not found in the deck. "+ f"You must enable model fallback in your .pipelex/pipelex.toml file to permit the following fallbacks, "+ f"or enable a backend that supports '{ideal_model_handle}'. "+ )+ raise ModelNotFoundError(message=msg, model_handle=model_handle)+ if inference_model := self.get_optional_inference_model(model_handle=fallback):+ if fallback_index > 0:+ # Only log if we haven't logged for this model_handle before+ if model_handle not in self._logged_fallback_warnings:+ # Waterfall success: we explain what happened in the logs+ msg = (+ f"Inference model fallback: '{ideal_model_handle}' was not found in the model deck, "+ f"so it was replaced by '{fallback}'. "+ f"As a consequence, the results of the workflow may not have the expected quality, "+ f"and the workflow might fail due to feature limitations such as context window size, etc. "+ f"Consider getting access to '{ideal_model_handle}'."+ )+ enabled_backends = {model.backend_name for model in self.inference_models.values()}+ if "pipelex_inference" not in enabled_backends and self._is_model_available_in_backend(+ model_handle=ideal_model_handle, backend_name="pipelex_inference"+ ):+ msg += (+ f" Note that many high quality models such as '{ideal_model_handle}' are available from the Pipelex Inference "+ f"platform and you can get free credits to try them out, please see our docs for more details about setting up "+ f"Pipelex Inference or other inference backends:\n{URLs.backend_provider_docs}"+ )+ else:+ msg += f" Please see our docs for more details about setting up inference backends:\n{URLs.backend_provider_docs}"+ log.info(msg)+ # Mark this warning as logged for this model_handle+ self._logged_fallback_warnings.add(model_handle)return inference_model- log.warning(f"Skipping model handle '{model_handle}' because it's not found in deck, it could be an external plugin.")+ msg = (+ f"Model handle '{model_handle}' is a waterfall (i.e. a list of models to try in order) "+ "but none of the fallback models were found in the model deck"+ )+ raise ModelWaterfallError(message=msg, model_handle=model_handle, fallback_list=fallback_list)+ log.verbose(f"Skipping model handle '{model_handle}' because it's was not found in the model deck, it could be an external plugin.")return Nonedef is_handle_defined(self, model_handle: str) -> bool:- return model_handle in self.inference_models or model_handle in self.aliases+ return model_handle in self.inference_models or model_handle in self.aliases or model_handle in self.waterfallsdef get_required_inference_model(self, model_handle: str) -> InferenceModelSpec:inference_model = self.get_optional_inference_model(model_handle=model_handle)if inference_model is None:msg = (- f"Model handle '{model_handle}' not found in deck. "- "Make sure its defined in the model decks '.pipelex/inference/deck/base_deck.toml' or '.pipelex/inference/deck/overrides.toml'"+ f"Model handle '{model_handle}' was not found in the model deck. "+ "Make sure it's defined in ond of the model decks '.pipelex/inference/deck/*.toml'. ""If the model handle is indeed in the deck, make sure the required backend for this model to run is enabled in ""'.pipelex/inference/backends.toml' and that you have the necessary credentials."- "To find what backend is required for this model, look at the routing profile in .pipelex/inference/routing_profiles.toml"+ "To find what backend is required for this model, look at the routing profile in .pipelex/inference/routing_profiles.toml. ""Learn more about the inference backend system in the Pipelex documentation: ""https://docs.pipelex.com/pages/configuration/config-technical/inference-backend-config/")- raise ModelNotFoundError(msg)+ raise ModelNotFoundError(message=msg, model_handle=model_handle)if model_handle not in self.inference_models:log.verbose(f"Model handle '{model_handle}' is an alias which resolves to '{inference_model.name}'")return inference_modeldiff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.pyindex 0efca195a..8cf96c6e9 100644--- a/pipelex/cogt/models/model_deck_check.py+++ b/pipelex/cogt/models/model_deck_check.py@@ -1,7 +1,8 @@-from pipelex.cogt.exceptions import ExtractChoiceNotFoundError, ImgGenChoiceNotFoundError, LLMChoiceNotFoundError+from pipelex.cogt.exceptions import ModelChoiceNotFoundErrorfrom pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSettingfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSettingfrom pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting+from pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.hub import get_model_deck@@ -13,8 +14,8 @@ def check_llm_choice_with_deck(llm_choice: LLMModelChoice):if llm_choice in model_deck.llm_presets or model_deck.is_handle_defined(model_handle=llm_choice):return- msg = f"LLM choice '{llm_choice}' not found in deck"- raise LLMChoiceNotFoundError(msg)+ msg = f"LLM choice '{llm_choice}' was not found in the model deck"+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.LLM, model_choice=llm_choice)def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):@@ -23,8 +24,8 @@ def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):model_deck = get_model_deck()if extract_choice in model_deck.extract_presets or model_deck.is_handle_defined(model_handle=extract_choice):return- msg = f"OCR choice '{extract_choice}' not found in deck"- raise ExtractChoiceNotFoundError(msg)+ msg = f"OCR choice '{extract_choice}' was not found in the model deck"+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.TEXT_EXTRACTOR, model_choice=extract_choice)def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):@@ -33,5 +34,5 @@ def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):model_deck = get_model_deck()if img_gen_choice in model_deck.img_gen_presets or model_deck.is_handle_defined(model_handle=img_gen_choice):return- msg = f"Image generation choice '{img_gen_choice}' not found in deck"- raise ImgGenChoiceNotFoundError(msg)+ msg = f"Image generation choice '{img_gen_choice}' was not found in the model deck"+ raise ModelChoiceNotFoundError(message=msg, model_type=ModelType.IMG_GEN, model_choice=img_gen_choice)diff --git a/pipelex/cogt/models/model_deck_config.py b/pipelex/cogt/models/model_deck_config.pynew file mode 100644index 000000000..04d4bcad3--- /dev/null+++ b/pipelex/cogt/models/model_deck_config.py@@ -0,0 +1,9 @@+from pydantic import Field++from pipelex.system.configuration.config_model import ConfigModel+from pipelex.system.runtime import ProblemReaction+++class ModelDeckConfig(ConfigModel):+ is_model_fallback_enabled: bool+ missing_presets_reaction: ProblemReaction = Field(strict=False)diff --git a/pipelex/cogt/models/model_deck_loader.py b/pipelex/cogt/models/model_deck_loader.pynew file mode 100644index 000000000..908404e3f--- /dev/null+++ b/pipelex/cogt/models/model_deck_loader.py@@ -0,0 +1,31 @@+from typing import Any++from pydantic import ValidationError++from pipelex.cogt.exceptions import ModelDeckNotFoundError, ModelDeckValidationError+from pipelex.cogt.models.model_deck import ModelDeckBlueprint+from pipelex.tools.misc.json_utils import deep_update+from pipelex.tools.misc.toml_utils import load_toml_from_path+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error+++def load_model_deck_blueprint(model_deck_paths: list[str]) -> ModelDeckBlueprint:+ full_deck_dict: dict[str, Any] = {}+ if not model_deck_paths:+ msg = "No Model deck paths found. Please run `pipelex init config` to create the set up the base deck."+ raise ModelDeckNotFoundError(msg)++ for deck_path in model_deck_paths:+ try:+ deck_dict = load_toml_from_path(path=deck_path)+ except FileNotFoundError as not_found_exc:+ msg = f"Could not find Model Deck file at '{deck_path}': {not_found_exc}"+ raise ModelDeckNotFoundError(msg) from not_found_exc+ deep_update(full_deck_dict, deck_dict)++ try:+ return ModelDeckBlueprint.model_validate(full_deck_dict)+ except ValidationError as exc:+ valiation_error_msg = format_pydantic_validation_error(exc)+ msg = f"Invalid Model Deck configuration in {model_deck_paths}: {valiation_error_msg}"+ raise ModelDeckValidationError(msg) from excdiff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.pyindex a8f4cff27..1ddb954ad 100644--- a/pipelex/cogt/models/model_manager.py+++ b/pipelex/cogt/models/model_manager.py@@ -1,25 +1,23 @@-from typing import Any--from pydantic import ValidationErrorfrom typing_extensions import override-from pipelex import log-from pipelex.cogt.exceptions import ModelDeckNotFoundError, ModelDeckValidationError, ModelManagerError+from pipelex.cogt.exceptions import ModelManagerErrorfrom pipelex.cogt.model_backends.backend import InferenceBackendfrom pipelex.cogt.model_backends.backend_library import InferenceBackendLibraryfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_routing.routing_models import BackendMatchingMethod-from pipelex.cogt.model_routing.routing_profile_library import RoutingProfileLibrary+from pipelex.cogt.model_routing.routing_profile import RoutingProfile+from pipelex.cogt.model_routing.routing_profile_loader import load_active_routing_profilefrom pipelex.cogt.models.model_deck import ModelDeck, ModelDeckBlueprint+from pipelex.cogt.models.model_deck_loader import load_model_deck_blueprintfrom pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract-from pipelex.config import get_config-from pipelex.tools.misc.json_utils import deep_update-from pipelex.tools.misc.toml_utils import load_toml_from_path+from pipelex.config import ConfigPaths, get_config+from pipelex.tools.misc.file_utils import find_files_in_dir+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractclass ModelManager(ModelManagerAbstract):def __init__(self) -> None:- self.routing_profile_library = RoutingProfileLibrary.make_empty()+ self._routing_profile: RoutingProfile | None = Noneself.inference_backend_library = InferenceBackendLibrary.make_empty()self.model_deck: ModelDeck | None = None@@ -30,50 +28,63 @@ class ModelManager(ModelManagerAbstract):raise RuntimeError(msg)return self.model_deck+ @classmethod+ def get_model_deck_paths(cls, deck_dir_path: str) -> list[str]:+ """Get all Model deck TOML file paths sorted alphabetically."""+ model_deck_paths = [+ str(path)+ for path in find_files_in_dir(+ dir_path=deck_dir_path,+ pattern="*.toml",+ is_recursive=True,+ )+ ]+ model_deck_paths.sort()+ return model_deck_paths+@overridedef teardown(self) -> None:- self.routing_profile_library.reset()+ self.model_deck = Noneself.inference_backend_library.reset()+ self._routing_profile = None@override- def setup(self) -> None:- self.routing_profile_library.load()- self.inference_backend_library.load()- deck_blueprint = self.load_deck_blueprint()- self.model_deck = self.build_deck(model_deck_blueprint=deck_blueprint)-- @classmethod- def load_deck_blueprint(cls) -> ModelDeckBlueprint:- deck_paths = get_config().cogt.inference_config.get_model_deck_paths()- full_deck_dict: dict[str, Any] = {}- if not deck_paths:- msg = "No Model deck paths found. Please run `pipelex init config` to create the set up the base deck."- raise ModelDeckNotFoundError(msg)+ def setup(self, secrets_provider: SecretsProviderAbstract) -> None:+ self.inference_backend_library.load(+ secrets_provider=secrets_provider,+ backends_library_path=ConfigPaths.BACKENDS_FILE_PATH,+ backends_dir_path=ConfigPaths.BACKENDS_DIR_PATH,+ )+ enabled_backends = self.inference_backend_library.all_enabled_backends()+ self._routing_profile = load_active_routing_profile(+ routing_profile_library_path=ConfigPaths.ROUTING_PROFILES_FILE_PATH,+ enabled_backends=enabled_backends,+ )+ model_deck_paths = ModelManager.get_model_deck_paths(deck_dir_path=ConfigPaths.MODEL_DECKS_DIR_PATH)+ deck_blueprint = load_model_deck_blueprint(model_deck_paths=model_deck_paths)+ self.model_deck = self.build_deck(enabled_backends=enabled_backends, model_deck_blueprint=deck_blueprint)- for deck_path in deck_paths:- try:- deck_dict = load_toml_from_path(path=deck_path)- except FileNotFoundError as not_found_exc:- msg = f"Could not find Model Deck file at '{deck_path}': {not_found_exc}"- raise ModelDeckNotFoundError(msg) from not_found_exc- deep_update(full_deck_dict, deck_dict)+ @override+ def validate_model_deck(self):+ self.get_model_deck().validate_registered_models()- try:- return ModelDeckBlueprint.model_validate(full_deck_dict)- except ValidationError as exc:- msg = f"Invalid Model Deck configuration in {deck_paths}: {exc}"- raise ModelDeckValidationError(msg) from exc+ @property+ def routing_profile(self) -> RoutingProfile:+ if self._routing_profile is None:+ msg = "No active routing profile loaded"+ raise RuntimeError(msg)+ return self._routing_profile- def build_deck(self, model_deck_blueprint: ModelDeckBlueprint) -> ModelDeck:+ def build_deck(self, enabled_backends: list[str], model_deck_blueprint: ModelDeckBlueprint) -> ModelDeck:all_models_and_possible_backends = self.inference_backend_library.get_all_models_and_possible_backends()inference_models: dict[str, InferenceModelSpec] = {}for model_name, available_backends in all_models_and_possible_backends.items():- backend_match_for_model = self.routing_profile_library.get_backend_match_for_model_from_active_routing_profile(+ backend_match_for_model = self.routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,model_name=model_name,)if backend_match_for_model is None:- log.verbose(f"No backend match found for model '{model_name}'")continuematched_backend_name = backend_match_for_model.backend_namebackend = self.inference_backend_library.get_inference_backend(backend_name=matched_backend_name)@@ -91,17 +102,22 @@ class ModelManager(ModelManagerAbstract):)raise ModelManagerError(msg)case BackendMatchingMethod.PATTERN_MATCH:- log.verbose(- f"Model spec '{model_name}' not found in backend '{matched_backend_name}' but it's OK because "- f"it was only matched by pattern in routing profile '{backend_match_for_model.routing_profile_name}'",- )# We can skip it because it was only a pattern matchcontinuecase BackendMatchingMethod.DEFAULT:# We could not find the model spec, but it was a default match,# so we can look for it in the other available backends- # TODO: enable to set the order or priority of the available backends- for available_backend in available_backends:+ # Use fallback_order if specified, then try remaining enabled backends+ if backend_match_for_model.fallback_order:+ # Try fallback_order first, then any enabled backends not in fallback_order+ backends_to_try = backend_match_for_model.fallback_order + [+ b for b in enabled_backends if b not in backend_match_for_model.fallback_order+ ]+ else:+ # No fallback_order specified, use all available_backends+ backends_to_try = available_backends++ for available_backend in backends_to_try:if available_backend == matched_backend_name:# we've already checked the matched_backend_name and it didn't have the model spec, that's why we're herecontinue@@ -113,16 +129,15 @@ class ModelManager(ModelManagerAbstract):if model_spec is not None:breakif model_spec is None:- msg = (- f"Model spec '{model_name}' not found in any of the available backends '{available_backends}' "- f"which was set as default in routing profile '{backend_match_for_model.routing_profile_name}'"- )- raise ModelManagerError(msg)+ # Model not available in any of the searched backends - skip it+ # Not all models need to be available in the configured backends+ continueinference_models[model_name] = model_specreturn ModelDeck(inference_models=inference_models,aliases=model_deck_blueprint.aliases,+ waterfalls=model_deck_blueprint.waterfalls,llm_presets=model_deck_blueprint.llm.presets,llm_choice_defaults=model_deck_blueprint.llm.choice_defaults,llm_choice_overrides=model_deck_blueprint.llm.choice_overrides,@@ -130,6 +145,7 @@ class ModelManager(ModelManagerAbstract):extract_choice_default=model_deck_blueprint.extract.choice_default,img_gen_presets=model_deck_blueprint.img_gen.presets,img_gen_choice_default=model_deck_blueprint.img_gen.choice_default,+ model_deck_config=get_config().cogt.model_deck_config,)@overridediff --git a/pipelex/cogt/models/model_manager_abstract.py b/pipelex/cogt/models/model_manager_abstract.pyindex b0cc94593..e97a35c64 100644--- a/pipelex/cogt/models/model_manager_abstract.py+++ b/pipelex/cogt/models/model_manager_abstract.py@@ -3,15 +3,20 @@ from abc import ABC, abstractmethodfrom pipelex.cogt.model_backends.backend import InferenceBackendfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.models.model_deck import ModelDeck+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractclass ModelManagerAbstract(ABC):+ @abstractmethod+ def validate_model_deck(self):+ pass+@abstractmethoddef teardown(self) -> None:pass@abstractmethod- def setup(self) -> None:+ def setup(self, secrets_provider: SecretsProviderAbstract) -> None:pass@abstractmethoddiff --git a/pipelex/config.py b/pipelex/config.pyindex f82415790..4937bb4ca 100644--- a/pipelex/config.py+++ b/pipelex/config.py@@ -17,6 +17,24 @@ from pipelex.tools.log.log_config import LogConfigfrom pipelex.types import StrEnum+class ConfigPaths:+ DEFAULT_CONFIG_DIR_PATH = "./.pipelex"+ INFERENCE_DIR_NAME = "inference"+ INFERENCE_DIR_PATH = f"{DEFAULT_CONFIG_DIR_PATH}/{INFERENCE_DIR_NAME}"+ BACKENDS_FILE_NAME = "backends.toml"+ BACKENDS_FILE_PATH = f"{INFERENCE_DIR_PATH}/{BACKENDS_FILE_NAME}"+ BACKENDS_DIR_NAME = "backends"+ BACKENDS_DIR_PATH = f"{INFERENCE_DIR_PATH}/{BACKENDS_DIR_NAME}"+ ROUTING_PROFILES_FILE_NAME = "routing_profiles.toml"+ ROUTING_PROFILES_FILE_PATH = f"{INFERENCE_DIR_PATH}/{ROUTING_PROFILES_FILE_NAME}"+ MODEL_DECKS_DIR_NAME = "deck"+ MODEL_DECKS_DIR_PATH = f"{INFERENCE_DIR_PATH}/{MODEL_DECKS_DIR_NAME}"+ BASE_DECK_FILE_NAME = "base_deck.toml"+ BASE_DECK_FILE_PATH = f"{MODEL_DECKS_DIR_PATH}/{BASE_DECK_FILE_NAME}"+ OVERRIDES_DECK_FILE_NAME = "overrides.toml"+ OVERRIDES_DECK_FILE_PATH = f"{MODEL_DECKS_DIR_PATH}/{OVERRIDES_DECK_FILE_NAME}"++class StaticValidationReaction(StrEnum):RAISE = "raise"LOG = "log"diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pyindex 551d87746..44bc6e0e5 100644--- a/pipelex/core/pipes/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -1,14 +1,15 @@from abc import ABC, abstractmethod-from typing import Any+from typing import Any, finalfrom pydantic import BaseModel, ConfigDict, Field, field_validator+from pipelex.cogt.exceptions import ModelChoiceNotFoundErrorfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.input_requirements import InputRequirementsfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprintfrom pipelex.core.pipes.pipe_output import PipeOutput-from pipelex.exceptions import PipeStackOverflowError+from pipelex.exceptions import PipeOperatorModelChoiceError, PipeStackOverflowErrorfrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -38,8 +39,22 @@ class PipeAbstract(ABC, BaseModel):def validate_output(self):"""Validate the output for the pipe."""+ @finaldef validate_with_libraries(self):"""Validate the pipe with the libraries, after the static validation"""+ try:+ self._validate_with_libraries()+ except ModelChoiceNotFoundError as exc:+ raise PipeOperatorModelChoiceError(+ message=exc.message,+ pipe_type=self.pipe_type,+ pipe_code=self.code,+ model_type=exc.model_type,+ model_choice=exc.model_choice,+ ) from exc++ def _validate_with_libraries(self):+ """Validate the pipe with the libraries, after the static validation"""@abstractmethoddef required_variables(self) -> set[str]:diff --git a/pipelex/core/validation_errors.py b/pipelex/core/validation_errors.pyindex 50a9ce948..d5e8566d7 100644--- a/pipelex/core/validation_errors.py+++ b/pipelex/core/validation_errors.py@@ -1,7 +1,9 @@-from typing_extensions import Protocol+from abc import ABC, abstractmethodfrom pipelex.builder.validation_error_data import ConceptDefinitionErrorData-class ValidationErrorDetailsProtocol(Protocol):- def get_concept_definition_errors(self) -> list[ConceptDefinitionErrorData]: ...+class PipelexValidationExceptionAbstract(Exception, ABC):+ @abstractmethod+ def get_concept_definition_errors(self) -> list[ConceptDefinitionErrorData]:+ passdiff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex ef83c534d..ac98bb825 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -12,11 +12,15 @@ from pipelex.builder.validation_error_data import (StaticValidationErrorType,SyntaxErrorData,)-from pipelex.core.validation_errors import ValidationErrorDetailsProtocol+from pipelex.core.validation_errors import PipelexValidationExceptionAbstractfrom pipelex.system.exceptions import RootExceptionfrom pipelex.tools.misc.context_provider_abstract import ContextProviderExceptionif TYPE_CHECKING:+ from pipelex.cogt.extract.extract_setting import ExtractModelChoice+ from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice+ from pipelex.cogt.llm.llm_setting import LLMModelChoice+ from pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.pipe_run.pipe_run_mode import PipeRunMode@@ -127,7 +131,7 @@ class LibraryError(PipelexException):pass-class LibraryLoadingError(LibraryError, ValidationErrorDetailsProtocol):+class LibraryLoadingError(LibraryError, PipelexValidationExceptionAbstract):"""Error raised when loading library components fails."""def __init__(@@ -377,6 +381,65 @@ class StructureClassError(PipelexException):pass+class PipeOperatorException(PipelexException):+ pass+++class PipeOperatorModelAvailabilityError(PipelexException):+ def __init__(+ self,+ message: str,+ run_mode: PipeRunMode,+ pipe_type: str,+ pipe_code: str,+ pipe_stack: list[str],+ model_handle: str,+ fallback_list: list[str] | None = None,+ ):+ self.run_mode = run_mode+ self.pipe_type = pipe_type+ self.pipe_code = pipe_code+ self.pipe_stack = pipe_stack+ self.model_handle = model_handle+ self.fallback_list = fallback_list+ super().__init__(message)+++class PipeOperatorModelChoiceError(PipelexException):+ def __init__(+ self,+ message: str,+ pipe_type: str,+ pipe_code: str,+ model_type: ModelType,+ model_choice: LLMModelChoice | ExtractModelChoice | ImgGenModelChoice,+ ):+ self.pipe_type = pipe_type+ self.pipe_code = pipe_code+ self.model_type = model_type+ self.model_choice = model_choice+ super().__init__(message)++ def desc(self) -> str:+ msg = f"{self.message}"+ msg += f" • pipe='{self.pipe_code}' ({self.pipe_type})"+ msg += f" • model_type='{self.model_type}'"++ # Extract the choice identifier from the model_choice union type+ if isinstance(self.model_choice, str):+ # It's a preset/alias string+ msg += f" • choice='{self.model_choice}'"+ else:+ # It's a Setting object with a model field and optional desc()+ msg += f" • choice={self.model_choice.desc()}"++ return msg++ @override+ def __str__(self) -> str:+ return self.desc()++class PipeControllerError(PipelexException):passdiff --git a/pipelex/hub.py b/pipelex/hub.pyindex 1af5c59b9..8e1015210 100644--- a/pipelex/hub.py+++ b/pipelex/hub.py@@ -90,14 +90,14 @@ class PipelexHub:# tools- def setup_config(self, config_cls: type[ConfigRoot], specific_config_path: str | None = None):+ def setup_config(self, config_cls: type[ConfigRoot]):"""Set the global configuration instance.# Args:# config (Config): The configuration instance to set."""- config = config_manager.load_config(specific_config_path)- self.set_config(config=config_cls.model_validate(config))+ config_dict = config_manager.load_config()+ self.set_config(config=config_cls.model_validate(config_dict))def set_config(self, config: ConfigRoot):if self._config is not None:diff --git a/pipelex/kit/agent_rules/llms.md b/pipelex/kit/agent_rules/llms.mdindex 1c3b01521..e8b5979ed 100644--- a/pipelex/kit/agent_rules/llms.md+++ b/pipelex/kit/agent_rules/llms.md@@ -48,7 +48,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/pipelex/kit/agent_rules/write_pipelex.md b/pipelex/kit/agent_rules/write_pipelex.mdindex 77405af0a..aa4110624 100644--- a/pipelex/kit/agent_rules/write_pipelex.md+++ b/pipelex/kit/agent_rules/write_pipelex.md@@ -817,7 +817,7 @@ Presets are meant to record the choice of an llm with its hyper parameters (tempExamples:```toml-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }+llm_to_engineer = { model = "base-claude", temperature = 1 }llm_to_extract_invoice = { model = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" }```diff --git a/pipelex/kit/configs/inference/backends.toml b/pipelex/kit/configs/inference/backends.tomlindex 027d9fee0..536f3675d 100644--- a/pipelex/kit/configs/inference/backends.toml+++ b/pipelex/kit/configs/inference/backends.toml@@ -16,6 +16,11 @@ enabled = trueendpoint = "https://inference.pipelex.com/v1"api_key = "${PIPELEX_INFERENCE_API_KEY}"+[anthropic]+enabled = false+api_key = "${ANTHROPIC_API_KEY}"+claude_4_tokens_limit = 8192+[azure_openai]display_name = "Azure OpenAI"enabled = false@@ -28,32 +33,42 @@ display_name = "Amazon Bedrock"enabled = falseaws_region = "${AWS_REGION}"+[blackboxai]+display_name = "BlackBox AI"+enabled = false+endpoint = "https://api.blackbox.ai/v1"+api_key = "${BLACKBOX_API_KEY}"++[fal]+display_name = "FAL"+enabled = false+api_key = "${FAL_API_KEY}"+[google]display_name = "Google AI"enabled = falseapi_key = "${GOOGLE_API_KEY}"-[vertexai]-display_name = "Google Vertex AI"+[mistral]+display_name = "Mistral AI"enabled = false-gcp_project_id = "${GCP_PROJECT_ID}"-gcp_location = "${GCP_LOCATION}"-gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"+api_key = "${MISTRAL_API_KEY}"++[ollama]+enabled = false+endpoint = "http://localhost:11434/v1"[openai]display_name = "OpenAI"enabled = falseapi_key = "${OPENAI_API_KEY}"-[anthropic]-enabled = false-api_key = "${ANTHROPIC_API_KEY}"-claude_4_tokens_limit = 8192--[mistral]-display_name = "Mistral AI"+[vertexai]+display_name = "Google Vertex AI"enabled = false-api_key = "${MISTRAL_API_KEY}"+gcp_project_id = "${GCP_PROJECT_ID}"+gcp_location = "${GCP_LOCATION}"+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"[xai]display_name = "xAI"@@ -61,20 +76,5 @@ enabled = falseendpoint = "https://api.x.ai/v1"api_key = "${XAI_API_KEY}"-[ollama]-enabled = false-endpoint = "http://localhost:11434/v1"--[blackboxai]-display_name = "BlackBox AI"-enabled = false-endpoint = "https://api.blackbox.ai/v1"-api_key = "${BLACKBOX_API_KEY}"--[fal]-display_name = "FAL"-enabled = false-api_key = "${FAL_API_KEY}"-[internal] # software-only backend, runs internally, without AIenabled = truediff --git a/pipelex/kit/configs/inference/backends/anthropic.toml b/pipelex/kit/configs/inference/backends/anthropic.tomlindex 4b57964c4..66721e2c1 100644--- a/pipelex/kit/configs/inference/backends/anthropic.toml+++ b/pipelex/kit/configs/inference/backends/anthropic.toml@@ -89,3 +89,11 @@ inputs = ["text", "images"]outputs = ["text", "structured"]max_prompt_images = 100costs = { input = 3.0, output = 15.0 }++["claude-4.5-haiku"]+model_id = "claude-haiku-4-5-20251001"+max_tokens = 64000+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 100+costs = { input = 1.0, output = 5.0 }diff --git a/pipelex/kit/configs/inference/backends/azure_openai.toml b/pipelex/kit/configs/inference/backends/azure_openai.tomlindex 135638ad5..930f161fd 100644--- a/pipelex/kit/configs/inference/backends/azure_openai.toml+++ b/pipelex/kit/configs/inference/backends/azure_openai.toml@@ -80,6 +80,12 @@ inputs = ["text"]outputs = ["text", "structured"]costs = { input = 1.1, output = 4.4 }+[o3]+model_id = "o3-2025-04-16"+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 2, output = 8 }+# --- GPT-5 Series -------------------------------------------------------------[gpt-5-mini]model_id = "gpt-5-mini-2025-08-07"diff --git a/pipelex/kit/configs/inference/backends/bedrock.toml b/pipelex/kit/configs/inference/backends/bedrock.tomlindex c27baf0b5..cea91344d 100644--- a/pipelex/kit/configs/inference/backends/bedrock.toml+++ b/pipelex/kit/configs/inference/backends/bedrock.toml@@ -92,8 +92,6 @@ outputs = ["text", "structured"]max_prompt_images = 100costs = { input = 3.0, output = 15.0 }-# anthropic.claude-sonnet-4-5-20250929-v1:0,Anthropic,arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0,-["claude-4.5-sonnet"]sdk = "bedrock_anthropic"model_id = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"@@ -102,3 +100,12 @@ inputs = ["text", "images"]outputs = ["text", "structured"]max_prompt_images = 100costs = { input = 3.0, output = 15.0 }++["claude-4.5-haiku"]+sdk = "bedrock_anthropic"+model_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0"+max_tokens = 8192+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 100+costs = { input = 1.0, output = 5.0 }diff --git a/pipelex/kit/configs/inference/backends/pipelex_inference.toml b/pipelex/kit/configs/inference/backends/pipelex_inference.tomlindex 805b761ad..e516e274e 100644--- a/pipelex/kit/configs/inference/backends/pipelex_inference.toml+++ b/pipelex/kit/configs/inference/backends/pipelex_inference.toml@@ -103,6 +103,12 @@ inputs = ["text", "images"]outputs = ["text", "structured"]costs = { input = 3, output = 15 }+["claude-4.5-haiku"]+model_id = "pipelex/claude-4.5-haiku"+inputs = ["text", "images"]+outputs = ["text", "structured"]+costs = { input = 1, output = 5 }+# --- Gemini LLMs --------------------------------------------------------------["gemini-2.0-flash"]model_id = "gemini/gemini-2.0-flash"diff --git a/pipelex/kit/configs/inference/deck/base_deck.toml b/pipelex/kit/configs/inference/deck/base_deck.tomlindex 3fcc50d11..12f516755 100644--- a/pipelex/kit/configs/inference/deck/base_deck.toml+++ b/pipelex/kit/configs/inference/deck/base_deck.toml@@ -28,27 +28,59 @@ best-gemini = "gemini-2.5-pro"best-mistral = "mistral-medium"best-grok = "grok-3"-cheap-gpt = "gpt-4o-mini"-cheap_llm_for_text = "gpt-4o-mini"-cheap_llm_for_object = "gpt-4o-mini"-cheap_llm_for_vision = "gemini-2.5-flash-lite"+# Image generation aliases+base-img-gen = "flux-pro/v1.1"+best-img-gen = "flux-pro/v1.1-ultra"+fast-img-gen = "fast-lightning-sdxl"-base_llm_for_text = "gpt-4o-mini"-base_llm_for_object = "gpt-4o-mini"+####################################################################################################+# Waterfalls+####################################################################################################+[waterfalls]smart_llm = ["claude-4.5-sonnet","claude-4.1-opus","claude-4-sonnet","gemini-2.5-pro",- "gpt-4o",+ "gpt-4o", # we're using gpt-4o here rather than gpt-5 until we'll handle reasoning parameters+ "grok-3",+]+smart_llm_with_vision = [+ "claude-4.5-sonnet",+ "claude-4.1-opus",+ "claude-4-sonnet",+ "gemini-2.5-pro",+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters+ "grok-3",+]+smart_llm_for_structured = [+ "claude-4.5-sonnet",+ "claude-4.1-opus",+ "claude-4-sonnet",+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters]-llm_for_large_codebase = ["gemini-2.5-pro", "claude-4.5-sonnet", "gpt-5"]-# Image generation aliases-base-img-gen = "flux-pro/v1.1"-best-img-gen = "flux-pro/v1.1-ultra"-fast-img-gen = "fast-lightning-sdxl"+llm_for_large_codebase = [+ "gemini-2.5-pro",+ "claude-4.5-sonnet",+ "gpt-4o", # we use gpt-4o here rather than gpt-5 until we handle reasoning parameters+ "gemini-2.5-flash",+ "grok-3",+]+cheap_llm = [+ "gpt-4o-mini",+ "gemini-2.5-flash-lite",+ "mistral-small",+ "claude-3-haiku",+ "grok-3-mini",+]+cheap_llm_for_vision = [+ "gpt-4o-mini",+ "gemini-2.5-flash-lite",+ "claude-3-haiku",+]+cheap_llm_for_structured = ["gpt-4o-mini", "mistral-small", "claude-3-haiku"]##################################################################################################### LLM Presets@@ -56,73 +88,47 @@ fast-img-gen = "fast-lightning-sdxl"[llm.presets]-####################################################################################################-# LLM Presets — General purpose--cheap_llm_for_text = { model = "cheap_llm_for_text", temperature = 0.5 }-cheap_llm_for_short_text = { model = "cheap_llm_for_text", temperature = 0.5, max_tokens = 50 }-cheap_llm_for_object = { model = "cheap_llm_for_object", temperature = 0.5 }-cheap_llm_to_structure = { model = "cheap_llm_for_object", temperature = 0.1 }--llm_for_testing_gen_text = { model = "base_llm_for_text", temperature = 0.5 }-llm_for_testing_gen_object = { model = "base_llm_for_object", temperature = 0.5 }-llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }-llm_for_testing_vision_structured = { model = "cheap-gpt", temperature = 0.5 }--####################################################################################################-# LLM Presets — Specific skills--# Cheap llm-llm_cheap = { model = "base-gemini", temperature = 0.1 }+# LLM Presets — Specific skills -------------------------------------------------------------# Generation skillsllm_for_factual_writing = { model = "base-gpt", temperature = 0.1 }llm_for_creative_writing = { model = "base-gpt", temperature = 0.9 }-llm_to_write_questions = { model = "gpt-4o-mini", temperature = 0.3 }--# Reasoning skills-llm_for_complex_reasoning = { model = "base-claude", temperature = 1 }-llm_to_reason_on_diagram = { model = "base-claude", temperature = 0.5 }-llm_to_analyze_data = { model = "base-claude", temperature = 0.5 }+llm_for_writing_cheap = { model = "gpt-4o-mini", temperature = 0.3 }-# Search and answer skills-llm_to_answer_easy_questions = { model = "gpt-4o-mini", temperature = 0.3 }-llm_to_answer_hard_questions = { model = "base-claude", temperature = 0.3 }+# Retrieve and answer questions skills+llm_to_answer_questions_cheap = { model = "gpt-4o-mini", temperature = 0.3 }+llm_to_answer_questions = { model = "base-claude", temperature = 0.3 }llm_to_retrieve = { model = "base-claude", temperature = 0.1 }-llm_for_enrichment = { model = "gpt-4o", temperature = 0.1 }-llm_to_enrich = { model = "base-claude", temperature = 0.1 }-llm_for_question_and_excerpt_reformulation = { model = "gpt-4o", temperature = 0.9 }# Engineering skills-llm_to_engineer = { model = "smart_llm", temperature = 0.2 }-llm_to_write = { model = "base-gemini", temperature = 0.2 }-llm_to_pipelex = { model = "base-claude", temperature = 0.2 }+llm_to_engineer = { model = "smart_llm_for_structured", temperature = 0.2 }llm_to_code = { model = "base-claude", temperature = 0.1 }llm_to_analyze_large_codebase = { model = "base-claude", temperature = 0.1 }-# Image skills-llm_for_basic_vision = { model = "gpt-4o-mini", temperature = 0.1 }-llm_to_write_img_gen_prompt = { model = "base-claude", temperature = 0.2 }-llm_to_describe_img = { model = "base-claude", temperature = 0.5 }-llm_for_visual_analysis = { model = "base-claude", temperature = 0.5 }-llm_for_visual_design = { model = "base-claude", temperature = 0.5 }-llm_to_design_fashion = { model = "base-claude", temperature = 0.7 }+# Vision skills+llm_for_img_to_text_cheap = { model = "gpt-4o-mini", temperature = 0.1 }llm_for_img_to_text = { model = "base-claude", temperature = 0.1 }+llm_for_diagram_to_text = { model = "base-claude", temperature = 0.3 }+llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }-# Extraction skills-llm_to_extract_diagram = { model = "base-claude", temperature = 0.5 }-llm_to_extract_invoice = { model = "claude-4.5-sonnet", temperature = 0.1 }-llm_to_extract_invoice_from_scan = { model = "base-claude", temperature = 0.5 }-llm_to_extract_legal_terms = { model = "base-claude", temperature = 0.1 }-llm_to_extract_tables = { model = "base-claude", temperature = 0.1 }+# Image generation prompting skills+llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }+llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }++# LLM Presets — For Testing ---------------------------------------------------------------------++llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }+llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.5 }+llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }+llm_for_testing_vision_structured = { model = "cheap_llm_for_vision", temperature = 0.5 }##################################################################################################### LLM Choices####################################################################################################[llm.choice_defaults]-for_text = "cheap_llm_for_text"-for_object = "cheap_llm_for_object"+for_text = "cheap_llm"+for_object = "cheap_llm_for_structured"##################################################################################################### Extract Presets@@ -144,7 +150,6 @@ choice_default = "gen_image_basic"[img_gen.presets]-# General purposegen_image_basic = { model = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }gen_image_fast = { model = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }diff --git a/pipelex/kit/configs/inference/routing_profiles.toml b/pipelex/kit/configs/inference/routing_profiles.tomlindex a5c5a0ce4..592533daf 100644--- a/pipelex/kit/configs/inference/routing_profiles.toml+++ b/pipelex/kit/configs/inference/routing_profiles.toml@@ -1,12 +1,12 @@# Routing profile library - Routes models to their backends-# ================================================+# =========================================================================================# This file controls which backend serves which model.# Simply change the 'active' field to switch profiles,# or you can add your own custom profiles.## Documentation: https://docs.pipelex.com# Support: https://go.pipelex.com/discord-# ================================================+# =========================================================================================# Which profile to use (change this to switch routing)active = "pipelex_first"@@ -21,17 +21,21 @@ active = "pipelex_first"# 3. The .pipelex/inference/backends.toml is already configured with api_key = "${PIPELEX_INFERENCE_API_KEY}"# which will get the key from the environment variable.-# ============================================+# =========================================================================================# Routing Profiles-# ============================================+# =========================================================================================[profiles.pipelex_first]description = "Use Pipelex Inference backend for all its supported models"default = "pipelex_inference"--[profiles.all_blackboxai]-description = "Use BlackBoxAI backend for all its supported models"-default = "blackboxai"+fallback_order = [+ "pipelex_inference",+ "azure_openai",+ "bedrock",+ "blackboxai",+ "mistral",+ "fal",+][profiles.pipelex_first.routes]# Pattern matching: "model-pattern" = "backend-name"@@ -39,33 +43,82 @@ default = "blackboxai""claude-*" = "pipelex_inference""grok-*" = "pipelex_inference""gemini-*" = "pipelex_inference"++[profiles.pipelex_first.optional_routes] # Each optional route is considered only if its backend is available"*-sdxl" = "blackboxai""flux-*" = "blackboxai""nano-banana" = "blackboxai""gpt-image-1" = "openai""mistral-ocr" = "mistral"+[profiles.all_anthropic]+description = "Use Anthropic backend for all its supported models"+default = "anthropic"++[profiles.all_azure_openai]+description = "Use Azure OpenAI backend for all its supported models"+default = "azure_openai"+[profiles.all_bedrock]description = "Use Bedrock backend for all its supported models"default = "bedrock"-[profiles.all_anthropic]-description = "Use Anthropic backend for all its supported models"-default = "anthropic"+[profiles.all_blackboxai]+description = "Use BlackBoxAI backend for all its supported models"+default = "blackboxai"++[profiles.all_fal]+description = "Use FAL backend for all its supported models"+default = "fal"-[profiles.all_gemini]+[profiles.all_google]description = "Use Google GenAI backend for all its supported models"default = "google"-[profiles.all_azure_openai]-description = "Use Azure OpenAI backend for all its supported models"-default = "azure_openai"+[profiles.all_mistral]+description = "Use Mistral backend for all its supported models"+default = "mistral"++[profiles.all_ollama]+description = "Use Ollama backend for all its supported models"+default = "ollama"++[profiles.all_openai]+description = "Use OpenAI backend for all its supported models"+default = "openai"++[profiles.all_vertexai]+description = "Use Vertex AI backend for all its supported models"+default = "vertexai"++[profiles.all_xai]+description = "Use xAI backend for all its supported models"+default = "xai"++# =========================================================================================+# Custom Profiles+# =========================================================================================+# Add your own profiles below following the same pattern:+#+# [profiles.your_profile_name]+# description = "What this profile does"+# default = "backend-name" # Where to route models by default+# [profiles.your_profile_name.routes]+# "model-pattern" = "backend-name" # Specific routing rules+#+# Pattern matching supports:+# - Exact names: "gpt-4o-mini"+# - Wildcards: "claude-*" (matches all models starting with claude-)+# - Partial wildcards: "*-sonnet" (matches all sonnet variants)-[profiles.custom_routing]-description = "Custom routing"+# =========================================================================================+# Example of a custom routing profile with mostly pattern matching and one specific model+# =========================================================================================+[profiles.example_routing_using_patterns]+description = "Example routing profile using patterns"default = "pipelex_inference"-[profiles.custom_routing.routes]+[profiles.example_routing_using_patterns.routes]# Pattern matching: "model-pattern" = "backend-name""gpt-*" = "azure_openai""claude-*" = "bedrock"@@ -75,10 +128,14 @@ default = "pipelex_inference""flux-*" = "fal""gpt-image-1" = "openai"-[profiles.to_test_various_backends]-description = "Route models to various backends"+# =========================================================================================+# Example of a custom routing profile with specific model matching+# =========================================================================================++[profiles.example_routing_using_specific_models]+description = "Example routing profile using specific models"-[profiles.to_test_various_backends.routes]+[profiles.example_routing_using_specific_models.routes]"gpt-5-nano" = "pipelex_inference""gpt-4o-mini" = "blackboxai""gpt-5-mini" = "openai"@@ -93,19 +150,3 @@ description = "Route models to various backends""grok-3" = "pipelex_inference""grok-3-mini" = "xai"--# ============================================-# Custom Profiles-# ============================================-# Add your own profiles below following the same pattern:-#-# [profiles.your_profile_name]-# description = "What this profile does"-# default = "backend-name" # Where to route models by default-# [profiles.your_profile_name.routes]-# "model-pattern" = "backend-name" # Specific routing rules-#-# Pattern matching supports:-# - Exact names: "gpt-4o-mini"-# - Wildcards: "claude-*" (matches all models starting with claude-)-# - Partial wildcards: "*-sonnet" (matches all sonnet variants)diff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.tomlindex 570dae25e..a1faa165f 100644--- a/pipelex/kit/configs/pipelex.toml+++ b/pipelex/kit/configs/pipelex.toml@@ -49,6 +49,13 @@ default_log_level = "INFO"[pipelex.log_config.package_log_levels]pipelex = "INFO"+[cogt]+[cogt.model_deck_config]+# Uncomment to disable model fallback: it will raise errors instead of using secondary model options+# is_model_fallback_enabled = false+# Uncomment to change the reaction to missing presets: "raise" (default), "log" or "none"+# missing_presets_reaction = "raise"+[cogt.llm_config]# Uncomment any of these to enable dumping the inputs or outputs of text-genration with an LLM# is_dump_text_prompts_enabled = truediff --git a/pipelex/kit/cursor_export.py b/pipelex/kit/cursor_export.pyindex ca207b8a9..ebc8b38d8 100644--- a/pipelex/kit/cursor_export.py+++ b/pipelex/kit/cursor_export.py@@ -7,7 +7,7 @@ import typerimport yamlfrom pipelex.kit.index_models import KitIndex-from pipelex.kit.paths import get_agents_dir+from pipelex.kit.paths import get_kit_agents_dirdef _iter_agent_files(agents_dir: Traversable) -> Iterable[tuple[str, str]]:@@ -52,7 +52,7 @@ def export_cursor_rules(repo_root: Path, idx: KitIndex, dry_run: bool = False) -idx: Kit index configurationdry_run: If True, only print what would be done"""- agents_dir = get_agents_dir()+ agents_dir = get_kit_agents_dir()out_dir = repo_root / ".cursor" / "rules"out_dir.mkdir(parents=True, exist_ok=True)@@ -76,7 +76,7 @@ def remove_cursor_rules(repo_root: Path, dry_run: bool = False) -> None:repo_root: Repository root directorydry_run: If True, only print what would be done"""- agents_dir = get_agents_dir()+ agents_dir = get_kit_agents_dir()out_dir = repo_root / ".cursor" / "rules"if not out_dir.exists():diff --git a/pipelex/kit/migrations/migrate_0.11.0_0.12.x.md b/pipelex/kit/migrations/migrate_0.11.0_0.12.x.mdindex 7959a4155..4877f2b6d 100644--- a/pipelex/kit/migrations/migrate_0.11.0_0.12.x.md+++ b/pipelex/kit/migrations/migrate_0.11.0_0.12.x.md@@ -682,13 +682,13 @@ default_outcome = "fail"**Before (.pipelex/inference/deck/base_deck.toml):**```toml[presets.llm]-llm_for_complex_reasoning = { llm_handle = "claude-3-5-sonnet", temperature = 1 }+llm_to_engineer = { llm_handle = "claude-3-5-sonnet", temperature = 1 }```**After:**```toml[presets.llm]-llm_for_complex_reasoning = { model = "claude-3-5-sonnet", temperature = 1 }+llm_to_engineer = { model = "claude-3-5-sonnet", temperature = 1 }```### Image generation presetsdiff --git a/pipelex/kit/migrations_export.py b/pipelex/kit/migrations_export.pyindex 792124310..a21a35df1 100644--- a/pipelex/kit/migrations_export.py+++ b/pipelex/kit/migrations_export.py@@ -2,7 +2,7 @@ from pathlib import Pathimport typer-from pipelex.kit.paths import get_migrations_dir+from pipelex.kit.paths import get_kit_migrations_dirdef export_migration_instructions(repo_root: Path, dry_run: bool = False) -> None:@@ -12,7 +12,7 @@ def export_migration_instructions(repo_root: Path, dry_run: bool = False) -> Nonrepo_root: Repository root directorydry_run: If True, only print what would be done"""- migrations_dir = get_migrations_dir()+ migrations_dir = get_kit_migrations_dir()out_dir = repo_root / ".pipelex" / "migrations"out_dir.mkdir(parents=True, exist_ok=True)diff --git a/pipelex/kit/paths.py b/pipelex/kit/paths.pyindex db49b0dbc..7374360d0 100644--- a/pipelex/kit/paths.py+++ b/pipelex/kit/paths.py@@ -11,7 +11,7 @@ def get_kit_root() -> Traversable:return files("pipelex.kit")-def get_agents_dir() -> Traversable:+def get_kit_agents_dir() -> Traversable:"""Get the agents directory within the kit package.Returns:@@ -20,7 +20,7 @@ def get_agents_dir() -> Traversable:return get_kit_root() / "agent_rules"-def get_configs_dir() -> Traversable:+def get_kit_configs_dir() -> Traversable:"""Get the configs directory within the kit package.Returns:@@ -29,7 +29,7 @@ def get_configs_dir() -> Traversable:return get_kit_root() / "configs"-def get_migrations_dir() -> Traversable:+def get_kit_migrations_dir() -> Traversable:"""Get the migrations directory within the kit package.Returns:diff --git a/pipelex/kit/targets_update.py b/pipelex/kit/targets_update.pyindex 3f80f530f..c4038b81f 100644--- a/pipelex/kit/targets_update.py+++ b/pipelex/kit/targets_update.py@@ -7,7 +7,7 @@ import typerfrom pipelex.kit.index_models import KitIndex, Targetfrom pipelex.kit.markers import find_span, replace_span, wrap-from pipelex.kit.paths import get_agents_dir+from pipelex.kit.paths import get_kit_agents_dirdef _read_agent_file(agents_dir: Traversable, name: str) -> str:@@ -58,7 +58,7 @@ def build_merged_rules(idx: KitIndex, agent_set: str | None = None) -> str:Returns:Merged markdown content with demoted headings"""- agents_dir = get_agents_dir()+ agents_dir = get_kit_agents_dir()if agent_set is None:agent_set = idx.agent_rules.default_setdiff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex cf1ff455d..dde79d4e4 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -340,14 +340,14 @@ class LibraryManager(LibraryManagerAbstract):try:blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()except FileNotFoundError as file_not_found_error:- msg = f"Could not find PLX blueprint at '{plx_file_path}'"+ msg = f"Could not find PLX bundle at '{plx_file_path}'"raise LibraryLoadingError(msg) from file_not_found_errorexcept PipeDefinitionError as pipe_def_error:- msg = f"Could not load PLX blueprint from '{plx_file_path}': {pipe_def_error}"+ msg = f"Could not load PLX bundle from '{plx_file_path}' because of: {pipe_def_error}"raise LibraryLoadingError(msg) from pipe_def_errorexcept ValidationError as validation_error:validation_error_msg = report_validation_error(category="plx", validation_error=validation_error)- msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {validation_error_msg}"+ msg = f"Could not load PLX bundle from '{plx_file_path}' because of: {validation_error_msg}"raise LibraryLoadingError(msg) from validation_errorblueprint.source = str(plx_file_path)blueprints.append(blueprint)diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex ff422d21a..820dc81b3 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -50,7 +50,7 @@ class PipeBatch(PipeController):return self@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):self._validate_required_variables()def _validate_required_variables(self) -> Self:diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex d5a637dfb..515853bd9 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -222,7 +222,7 @@ class PipeCondition(PipeController):raise extraneous_input_var_error@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):"""Perform full validation after all libraries are loaded.This is called after all pipes and concepts are available."""diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.pyindex 060e4777d..784e4e2f2 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py@@ -162,7 +162,7 @@ class PipeParallel(PipeController):raise extraneous_input_var_error@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):"""Perform full validation after all libraries are loaded.This is called after all pipes and concepts are available."""diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.pyindex abeb6ee5b..075954f0b 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py@@ -168,7 +168,7 @@ of sequence pipe '{self.code}' is not compatible with the output concept '{self.raise extraneous_input_var_error@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):"""Perform full validation after all libraries are loaded.This is called after all pipes and concepts are available."""diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.pyindex 67675a2cd..8cda93e4f 100644--- a/pipelex/pipe_operators/compose/pipe_compose.py+++ b/pipelex/pipe_operators/compose/pipe_compose.py@@ -77,7 +77,7 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):pass@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):pass@overridediff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.pyindex e32276a51..da32dd25d 100644--- a/pipelex/pipe_operators/extract/pipe_extract.py+++ b/pipelex/pipe_operators/extract/pipe_extract.py@@ -61,7 +61,7 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):return self@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):self._validate_inputs()if self.extract_choice:check_extract_choice_with_deck(extract_choice=self.extract_choice)diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.pyindex 4d2fcfa06..00447130d 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py@@ -98,7 +98,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):return self@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):self._validate_inputs()if self.img_gen:check_img_gen_choice_with_deck(img_gen_choice=self.img_gen)diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 5c63350a3..e8e8faaf2 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -80,11 +80,11 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):return self@override- def validate_with_libraries(self):+ def _validate_with_libraries(self):self.validate_inputs()self.llm_prompt_spec.validate_with_libraries()if self.llm_choices:- for llm_choice in self.llm_choices.list_choices():+ for llm_choice in self.llm_choices.list_choice_strings():check_llm_choice_with_deck(llm_choice=llm_choice)@overridediff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.pyindex dec6d33fb..67aa04255 100644--- a/pipelex/pipe_operators/pipe_operator.py+++ b/pipelex/pipe_operators/pipe_operator.py@@ -4,10 +4,12 @@ from typing import Generic, Literal, TypeVarfrom typing_extensions import overridefrom pipelex import log, pretty_print_md+from pipelex.cogt.exceptions import ModelNotFoundError, ModelWaterfallErrorfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.text_content import TextContent+from pipelex.exceptions import PipeOperatorModelAvailabilityErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -39,41 +41,61 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):)job_metadata.update(updated_metadata=updated_metadata)- match pipe_run_params.run_mode:- case PipeRunMode.LIVE:- if self.class_name not in ["PipeCompose", "PipeLLMPrompt"]:- name = f"Running [cyan]{self.class_name}[/cyan]"+ try:+ match pipe_run_params.run_mode:+ case PipeRunMode.LIVE:+ if self.class_name not in ["PipeCompose", "PipeLLMPrompt"]:+ name = f"Running [cyan]{self.class_name}[/cyan]"+ indent_level = len(pipe_run_params.pipe_stack) - 1+ indent = " " * indent_level+ label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name} → [green]{self.code}[/green]"+ log.info(f"{label} → [red]{self.output.code}[/red]")+ pipe_output = await self._run_operator_pipe(+ job_metadata=job_metadata,+ working_memory=working_memory,+ pipe_run_params=pipe_run_params,+ output_name=output_name,+ )+ if isinstance(pipe_output.main_stuff.content, TextContent):+ print()+ pretty_print_md(pipe_output.main_stuff_as_str, title=f"PipeOutput of pipe {self.code}")+ print()+ else:+ print()+ pipe_output.main_stuff.pretty_print_stuff(title=f"PipeOutput of pipe {self.code}: {self.output.code}")+ print()+ case PipeRunMode.DRY:+ name = f"Dry run [cyan]{self.class_name}[/cyan]"indent_level = len(pipe_run_params.pipe_stack) - 1indent = " " * indent_level- label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name} → [green]{self.code}[/green]"+ label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name}: [green]{self.code}[/green]"log.info(f"{label} → [red]{self.output.code}[/red]")- pipe_output = await self._run_operator_pipe(- job_metadata=job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- output_name=output_name,- )- if isinstance(pipe_output.main_stuff.content, TextContent):- print()- pretty_print_md(pipe_output.main_stuff_as_str, title=f"PipeOutput of pipe {self.code}")- print()- else:- print()- pipe_output.main_stuff.pretty_print_stuff(title=f"PipeOutput of pipe {self.code}: {self.output.code}")- print()- case PipeRunMode.DRY:- name = f"Dry run [cyan]{self.class_name}[/cyan]"- indent_level = len(pipe_run_params.pipe_stack) - 1- indent = " " * indent_level- label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name}: [green]{self.code}[/green]"- log.info(f"{label} → [red]{self.output.code}[/red]")- pipe_output = await self._dry_run_operator_pipe(- job_metadata=job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- output_name=output_name,- )-+ pipe_output = await self._dry_run_operator_pipe(+ job_metadata=job_metadata,+ working_memory=working_memory,+ pipe_run_params=pipe_run_params,+ output_name=output_name,+ )+ except ModelWaterfallError as model_waterfall_error:+ raise PipeOperatorModelAvailabilityError(+ message=model_waterfall_error.message,+ run_mode=pipe_run_params.run_mode,+ pipe_type=self.class_name,+ pipe_code=self.code,+ pipe_stack=pipe_run_params.pipe_stack,+ model_handle=model_waterfall_error.model_handle,+ fallback_list=model_waterfall_error.fallback_list,+ ) from model_waterfall_error+ except ModelNotFoundError as model_not_found_error:+ raise PipeOperatorModelAvailabilityError(+ message=model_not_found_error.message,+ run_mode=pipe_run_params.run_mode,+ pipe_type=self.class_name,+ pipe_code=self.code,+ pipe_stack=pipe_run_params.pipe_stack,+ model_handle=model_not_found_error.model_handle,+ fallback_list=None,+ ) from model_not_found_errorpipe_run_params.pop_pipe_from_stack(pipe_code=self.code)return pipe_outputdiff --git a/pipelex/pipe_run/dry_run.py b/pipelex/pipe_run/dry_run.pyindex d9459627b..e0c0971aa 100644--- a/pipelex/pipe_run/dry_run.py+++ b/pipelex/pipe_run/dry_run.py@@ -100,13 +100,28 @@ async def dry_run_pipes(pipes: list[PipeAbstract], run_in_parallel: bool = True,"""Parallel execution using ThreadPoolExecutor"""loop = asyncio.new_event_loop()asyncio.set_event_loop(loop)- return loop.run_until_complete(dry_run_pipe(pipe, raise_on_failure=raise_on_failure))+ try:+ return loop.run_until_complete(dry_run_pipe(pipe, raise_on_failure=raise_on_failure))+ finally:+ loop.close()with ThreadPoolExecutor() as executor:futures = [asyncio.get_running_loop().run_in_executor(executor, functools.partial(run_pipe_in_thread, pipe)) for pipe in pipes]for future in asyncio.as_completed(futures):- output = await future- results[output.pipe_code] = output+ try:+ output = await future+ results[output.pipe_code] = output+ except Exception as exc:+ # If raise_on_failure is True, re-raise the first exception encountered+ # Otherwise, this shouldn't happen as dry_run_pipe should return a DryRunOutput+ if raise_on_failure:+ # Cancel remaining futures+ for f in futures:+ if not f.done():+ f.cancel()+ raise+ # This path shouldn't normally be reached, but handle it gracefully+ log.error(f"Unexpected exception in dry run: {exc}")else:for pipe in pipes:results[pipe.code] = await dry_run_pipe(pipe, raise_on_failure=raise_on_failure)diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 705e04cd2..f5145b725 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -18,13 +18,13 @@ from pipelex.cogt.exceptions import (InferenceBackendLibraryValidationError,ModelDeckNotFoundError,ModelDeckValidationError,+ RoutingProfileDisabledBackendError,RoutingProfileLibraryNotFoundError,- RoutingProfileValidationError,)from pipelex.cogt.inference.inference_manager import InferenceManagerfrom pipelex.cogt.models.model_manager import ModelManagerfrom pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract-from pipelex.config import PipelexConfig, get_config+from pipelex.config import ConfigPaths, PipelexConfig, get_configfrom pipelex.core.concepts.concept_library import ConceptLibraryfrom pipelex.core.domains.domain_library import DomainLibraryfrom pipelex.core.pipes.pipe_library import PipeLibrary@@ -53,7 +53,7 @@ from pipelex.system.environment import get_optional_envfrom pipelex.system.registries.func_registry import func_registryfrom pipelex.system.runtime import IntegrationMode, runtime_managerfrom pipelex.system.telemetry.observer_telemetry import ObserverTelemetry-from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryConfig+from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TelemetryConfig, TelemetryModefrom pipelex.system.telemetry.telemetry_manager import DO_NOT_TRACK_ENV_VAR_KEY, TelemetryManagerfrom pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract, TelemetryManagerNoOpfrom pipelex.test_extras.registry_test_models import TestRegistryModels@@ -71,10 +71,10 @@ PACKAGE_NAME, PACKAGE_VERSION = get_package_info()class Pipelex(metaclass=MetaSingleton):def __init__(self,- config_dir_path: str = "./pipelex",+ config_dir_path: str | None = None,config_cls: type[ConfigRoot] | None = None,) -> None:- self.config_dir_path = config_dir_path+ self.config_dir_path = config_dir_path or ConfigPaths.DEFAULT_CONFIG_DIR_PATHself.pipelex_hub = PipelexHub()set_pipelex_hub(self.pipelex_hub)@@ -119,7 +119,7 @@ class Pipelex(metaclass=MetaSingleton):log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")@staticmethod- def _get_config_not_found_error_msg(component_name: str) -> str:+ def _get_config_file_not_found_error_msg(component_name: str) -> str:"""Generate error message for missing config files."""return f"Config files are missing for the {component_name}. Run `pipelex init config` to generate the missing files."@@ -129,10 +129,10 @@ class Pipelex(metaclass=MetaSingleton):msg = ""cause_exc = validation_exc.__cause__if cause_exc is None:- msg += f"\nUnxpexted cause:{cause_exc}"+ msg += f"\nUnexpexted error:{cause_exc}"raise PipelexSetupError(msg) from cause_excif not isinstance(cause_exc, ValidationError):- msg += f"\nUnxpexted cause:{cause_exc}"+ msg += f"\nUnexpexted cause:{cause_exc}"raise PipelexSetupError(msg) from cause_excreport = report_validation_error(category="config", validation_error=cause_exc)return f"""{msg}@@ -170,7 +170,8 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.self.class_registry = class_registry or ClassRegistry()self.pipelex_hub.set_class_registry(self.class_registry)self.kajson_manager = KajsonManager(class_registry=self.class_registry)- self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())+ secrets_provider = secrets_provider or EnvSecretsProvider()+ self.pipelex_hub.set_secrets_provider(secrets_provider=secrets_provider)self.pipelex_hub.set_storage_provider(storage_provider)# cogt@@ -180,19 +181,20 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.self.pipelex_hub.set_models_manager(models_manager=self.models_manager)try:- self.models_manager.setup()+ self.models_manager.setup(secrets_provider=secrets_provider)except RoutingProfileLibraryNotFoundError as routing_not_found_exc:- msg = self._get_config_not_found_error_msg("routing profile library")+ msg = self._get_config_file_not_found_error_msg("routing profile library")raise PipelexSetupError(msg) from routing_not_found_excexcept InferenceBackendLibraryNotFoundError as backend_not_found_exc:- msg = self._get_config_not_found_error_msg("inference backend library")+ msg = self._get_config_file_not_found_error_msg("inference backend library")raise PipelexSetupError(msg) from backend_not_found_excexcept ModelDeckNotFoundError as deck_not_found_exc:- msg = self._get_config_not_found_error_msg("model deck")+ msg = self._get_config_file_not_found_error_msg("model deck")raise PipelexSetupError(msg) from deck_not_found_exc- except RoutingProfileValidationError as routing_validation_exc:- msg = self._get_validation_error_msg("routing profile library", routing_validation_exc)- raise PipelexSetupError(msg) from routing_validation_exc+ except RoutingProfileDisabledBackendError as routing_profile_exc:+ msg = f"Some backend(s) required for a routing profile is not enabled: {routing_profile_exc}"+ raise PipelexSetupError(msg) from routing_profile_exc+except InferenceBackendLibraryValidationError as backend_validation_exc:msg = self._get_validation_error_msg("inference backend library", backend_validation_exc)raise PipelexSetupError(msg) from backend_validation_exc@@ -200,6 +202,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.msg = self._get_validation_error_msg("model deck", deck_validation_exc)msg += "\n\nIf you added your own config files to the model deck then you'll have to change them manually."raise PipelexSetupError(msg) from deck_validation_exc+except InferenceBackendCredentialsError as credentials_exc:backend_name = credentials_exc.backend_namevar_name = credentials_exc.key_name@@ -257,11 +260,16 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.telemetry_config_toml = load_toml_from_path(path=config_path)telemetry_config = TelemetryConfig.model_validate(telemetry_config_toml)- if telemetry_config.respect_dnt and (dnt := get_optional_env(DO_NOT_TRACK_ENV_VAR_KEY)) and dnt.lower() not in ["false", "0"]:- self.telemetry_manager = TelemetryManagerNoOp()- log.debug(f"Telemetry is disabled by env var 'DO_NOT_TRACK' which is set to {dnt}")- else:- self.telemetry_manager = telemetry_manager or TelemetryManager(telemetry_config=telemetry_config)+ match telemetry_config.telemetry_mode:+ case TelemetryMode.OFF:+ self.telemetry_manager = TelemetryManagerNoOp()+ log.debug("Telemetry is disabled because telemetry_mode is set to 'off'")+ case TelemetryMode.ANONYMOUS | TelemetryMode.IDENTIFIED:+ if telemetry_config.respect_dnt and (dnt := get_optional_env(DO_NOT_TRACK_ENV_VAR_KEY)) and dnt.lower() not in ["false", "0"]:+ self.telemetry_manager = TelemetryManagerNoOp()+ log.debug(f"Telemetry is disabled by env var 'DO_NOT_TRACK' which is set to {dnt}")+ else:+ self.telemetry_manager = telemetry_manager or TelemetryManager(telemetry_config=telemetry_config)else:self.telemetry_manager = TelemetryManagerNoOp()log.verbose(f"Telemetry is disabled because the integration mode '{integration_mode}' does not allow it")@@ -399,6 +407,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.**kwargs,)pipelex_instance.setup_libraries()+ pipelex_instance.models_manager.validate_model_deck()log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} ready")return pipelex_instancediff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex b24454d64..ed8b85fb2 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -93,13 +93,9 @@ api_key_method = "env"[cogt]-[cogt.inference_config]-inference_config_path = ".pipelex/inference"--[cogt.inference_manager_config]-is_auto_setup_preset_llm = true-is_auto_setup_preset_img_gen = true-is_auto_setup_preset_extract = true+[cogt.model_deck_config]+is_model_fallback_enabled = true+missing_presets_reaction = "log"[cogt.llm_config]default_max_images = 100diff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.pyindex f662192ba..9c7d18981 100644--- a/pipelex/plugins/openai/openai_llm_worker.py+++ b/pipelex/plugins/openai/openai_llm_worker.py@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Anyimport instructorimport openaifrom instructor.exceptions import InstructorRetryException-from openai import NOT_GIVEN, APIConnectionError, BadRequestError, NotFoundError+from openai import NOT_GIVEN, APIConnectionError, BadRequestError, NotFoundError, omitif TYPE_CHECKING:from openai.types.chat import ChatCompletionMessage@@ -84,7 +84,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):response = await self.openai_client_for_text.chat.completions.create(model=self.inference_model.model_id,temperature=temperature,- max_tokens=llm_job.job_params.max_tokens or None,+ max_tokens=llm_job.job_params.max_tokens or omit,seed=llm_job.job_params.seed,messages=messages,)diff --git a/pipelex/system/configuration/config_loader.py b/pipelex/system/configuration/config_loader.pyindex 5b009aac5..0b524df54 100644--- a/pipelex/system/configuration/config_loader.py+++ b/pipelex/system/configuration/config_loader.py@@ -1,4 +1,3 @@-import importlibimport osfrom typing import Any@@ -72,39 +71,7 @@ class ConfigLoader:config_path = os.path.join(self.pipelex_config_dir, CONFIG_NAME)return load_toml_from_path_if_exists(config_path) or {}- def load_inheritance_config(self, the_pipelex_config: dict[str, Any]):- """Load the config by inheritance in a pyproject.toml file.- This will be removed in the future.- Requires to have a pyproject.toml file in the project root.- """- pyproject_path = os.path.join(os.getcwd(), "pyproject.toml")- if not os.path.exists(pyproject_path):- print(f"pyproject.toml not found in {os.getcwd()}")- return-- def _find_package_path(package_name: str) -> str | None:- """Find package path by importing it"""- try:- module = importlib.import_module(package_name)- if hasattr(module, "__file__") and module.__file__:- return os.path.dirname(module.__file__)- except ImportError:- pass-- return None-- pyproject = load_toml_from_path(path=pyproject_path)- if "tool" in pyproject and "pipelex" in pyproject["tool"] and "config_inheritance" in pyproject["tool"]["pipelex"]:- for config_name in pyproject["tool"]["pipelex"]["config_inheritance"]:- package_path = _find_package_path(config_name)- if package_path:- config_path = os.path.join(package_path, "pipelex.toml")- if os.path.exists(config_path):- config = load_toml_from_path_if_exists(config_path)- if config:- deep_update(the_pipelex_config, config)-- def load_config(self, specific_config_path: str | None = None) -> dict[str, Any]:+ def load_config(self) -> dict[str, Any]:"""Load and merge configurations from pipelex and local config files.The configuration is loaded and merged in the following order:@@ -119,20 +86,16 @@ class ConfigLoader:Dict[str, Any]: The merged configuration dictionary"""- #################### 1. Load pipelex config ####################+ #################### 1. Load pipelex base config ####################pipelex_config = self.get_pipelex_config()- #################### 2. Load inheritance config for internal use ####################- # TODO: Undocumented feature, soon to be removed.- self.load_inheritance_config(pipelex_config)-- #################### 3. Load local (current project) config ####################+ #################### 2. Load local (current project) config ####################if not self.is_in_pipelex_config:local_config = self.get_local_config()if local_config:deep_update(pipelex_config, local_config)- #################### 4. Load overrides for the current project ####################+ #################### 3. Load overrides for the current project ####################list_of_overrides = [*CONFIG_BASE_OVERRIDES_BEFORE_ENV,runtime_manager.environment,@@ -148,15 +111,6 @@ class ConfigLoader:if override_dict := load_toml_from_path_if_exists(override_path):deep_update(pipelex_config, override_dict)- #################### 5. Load specific config ####################- if specific_config_path:- config = load_toml_from_path_if_exists(specific_config_path)- if config:- deep_update(pipelex_config, config)- else:- msg = f"Failed to load specific config from {specific_config_path}"- raise ConfigError(msg)-return pipelex_configdiff --git a/pipelex/system/telemetry/telemetry_manager.py b/pipelex/system/telemetry/telemetry_manager.pyindex 59c83a5eb..32bfefbf3 100644--- a/pipelex/system/telemetry/telemetry_manager.py+++ b/pipelex/system/telemetry/telemetry_manager.py@@ -1,4 +1,5 @@-from typing import Any, Callable+from contextlib import contextmanager+from typing import Any, Callable, Generatorimport posthogfrom posthog import Posthog, new_context, tag@@ -23,7 +24,7 @@ class TelemetryManager(TelemetryManagerAbstract):self.telemetry_config = telemetry_config# Create PostHog client- self.posthog = Posthog(+ self.posthog_client = Posthog(project_api_key=self.telemetry_config.project_api_key,host=self.telemetry_config.host,disable_geoip=not self.telemetry_config.geoip_enabled,@@ -32,13 +33,13 @@ class TelemetryManager(TelemetryManagerAbstract):)# Store original capture_exception method- self._original_capture_exception: Callable[..., Any] = self.posthog.capture_exception+ self._original_capture_exception: Callable[..., Any] = self.posthog_client.capture_exception# Wrap capture_exception to sanitize before sendingself._wrap_capture_exception()posthog.privacy_mode = True- posthog.default_client = self.posthog+ posthog.default_client = self.posthog_clientdef _handle_transmission_error(self, error: Exception | None, _items: list[dict[str, Any]]) -> None:"""Handle errors that occur during telemetry transmission.@@ -82,7 +83,7 @@ class TelemetryManager(TelemetryManagerAbstract):return self._original_capture_exception(exception, **kwargs)# Replace the method- self.posthog.capture_exception = sanitized_capture_exception # type: ignore[method-assign]+ self.posthog_client.capture_exception = sanitized_capture_exception # type: ignore[method-assign]@overridedef setup(self, integration_mode: IntegrationMode):@@ -92,7 +93,7 @@ class TelemetryManager(TelemetryManagerAbstract):tag(name=EventProperty.INTEGRATION, value=integration_mode)tag(name=EventProperty.PIPELEX_VERSION, value=package_version)tag(name=EventProperty.SETTING, value=Setting.TELEMETRY_MODE)- self.posthog.capture(+ self.posthog_client.capture(EventName.TELEMETRY_JUST_ENABLED,properties={EventProperty.TELEMETRY_MODE: telemetry_mode,@@ -102,7 +103,14 @@ class TelemetryManager(TelemetryManagerAbstract):@overridedef teardown(self):- pass+ if self.posthog_client:+ try:+ # PostHog client has a shutdown method to flush pending events+ # and close background threads+ self.posthog_client.shutdown()+ except Exception as exc:+ # Suppress any shutdown errors to avoid cascading failures+ log.debug(f"Error during PostHog shutdown: {exc}")@overridedef track_event(self, event_name: EventName, properties: dict[EventProperty, Any] | None = None):@@ -127,7 +135,7 @@ class TelemetryManager(TelemetryManagerAbstract):log.verbose(f"Telemetry is off, skipping event '{event_name}'")def _track_anonymous_event(self, event_name: str, properties: dict[str, Any]):- if not self.posthog:+ if not self.posthog_client:returnif self.telemetry_config.dry_mode_enabled:if properties:@@ -136,11 +144,11 @@ class TelemetryManager(TelemetryManagerAbstract):log.debug(f"Tracking anonymous event '{event_name}'. No properties.")else:properties["$process_person_profile"] = False- self.posthog.capture(event_name, properties=properties)+ self.posthog_client.capture(event_name, properties=properties)log.verbose(f"Tracked anonymous event '{event_name}' with properties: {properties}")def _track_identified_event(self, event_name: str, properties: dict[str, Any], user_id: str):- if not self.posthog:+ if not self.posthog_client:returnif self.telemetry_config.dry_mode_enabled:if properties:@@ -148,5 +156,12 @@ class TelemetryManager(TelemetryManagerAbstract):else:log.debug(f"Tracking identified event '{event_name}'. No properties.")else:- self.posthog.capture(event_name, distinct_id=user_id, properties=properties)+ self.posthog_client.capture(event_name, distinct_id=user_id, properties=properties)log.verbose(f"Tracked identified event '{event_name}' with properties: {properties}")++ @override+ @contextmanager+ def telemetry_context(self) -> Generator[None, None, None]:+ """Context manager that uses PostHog's new_context when telemetry is enabled."""+ with new_context():+ yielddiff --git a/pipelex/system/telemetry/telemetry_manager_abstract.py b/pipelex/system/telemetry/telemetry_manager_abstract.pyindex 529294468..b57403c88 100644--- a/pipelex/system/telemetry/telemetry_manager_abstract.py+++ b/pipelex/system/telemetry/telemetry_manager_abstract.py@@ -1,5 +1,6 @@from abc import ABC, abstractmethod-from typing import Any+from contextlib import contextmanager+from typing import Any, Generatorfrom typing_extensions import override@@ -30,6 +31,11 @@ class TelemetryManagerAbstract(ABC):def track_event(self, event_name: EventName, properties: dict[EventProperty, Any] | None = None):pass+ @abstractmethod+ @contextmanager+ def telemetry_context(self) -> Generator[None, None, None]:+ """Safe context manager for telemetry that works whether telemetry is enabled or not."""+class TelemetryManagerNoOp(TelemetryManagerAbstract):@override@@ -43,3 +49,9 @@ class TelemetryManagerNoOp(TelemetryManagerAbstract):@overridedef track_event(self, event_name: EventName, properties: dict[EventProperty, Any] | None = None):pass++ @override+ @contextmanager+ def telemetry_context(self) -> Generator[None, None, None]:+ """No-op context manager that doesn't use PostHog."""+ yielddiff --git a/pipelex/tools/secrets/secrets_utils.py b/pipelex/tools/secrets/secrets_utils.pyindex 11d9ea8c8..9d5d4972a 100644--- a/pipelex/tools/secrets/secrets_utils.py+++ b/pipelex/tools/secrets/secrets_utils.py@@ -1,9 +1,9 @@import re-from pipelex.hub import get_secrets_providerfrom pipelex.system.environment import EnvVarNotFoundError, get_optional_env, get_required_envfrom pipelex.system.exceptions import ToolExceptionfrom pipelex.tools.secrets.secrets_errors import SecretNotFoundError+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractfrom pipelex.types import StrEnum@@ -32,7 +32,7 @@ class VarPrefix(StrEnum):SECRET = "secret"-def substitute_vars(content: str) -> str:+def substitute_vars(content: str, secrets_provider: SecretsProviderAbstract) -> str:"""Substitute variable placeholders with values from environment variables or secrets.Supports the following placeholder formats:@@ -43,6 +43,7 @@ def substitute_vars(content: str) -> str:Args:content: Text content with variable placeholders+ secrets_provider: The secrets provider to use for secret lookupsReturns:Content with variables substituted@@ -57,7 +58,7 @@ def substitute_vars(content: str) -> str:# Check if it's a fallback pattern (contains |)if "|" in var_spec:- return _handle_fallback_pattern(var_spec)+ return _handle_fallback_pattern(var_spec, secrets_provider)# Check if it has a prefix (env: or secret:)if ":" in var_spec:@@ -77,10 +78,10 @@ def substitute_vars(content: str) -> str:case VarPrefix.ENV:return _get_env_var(var_name)case VarPrefix.SECRET:- return _get_secret(var_name)+ return _get_secret(var_name, secrets_provider)else:# Default behavior: use secrets provider- return _get_secret(var_spec)+ return _get_secret(var_spec, secrets_provider)# Pattern matches ${VAR_NAME} or ${prefix:VAR_NAME} or ${env:VAR|secret:VAR}# Restrict to not match across newlines, quotes, or nested braces@@ -88,7 +89,7 @@ def substitute_vars(content: str) -> str:return re.sub(pattern, replace_var, content)-def _handle_fallback_pattern(var_spec: str) -> str:+def _handle_fallback_pattern(var_spec: str, secrets_provider: SecretsProviderAbstract) -> str:"""Handle fallback pattern like 'env:VAR|secret:VAR'."""parts = [part.strip() for part in var_spec.split("|")]@@ -113,13 +114,13 @@ def _handle_fallback_pattern(var_spec: str) -> str:return valuecase VarPrefix.SECRET:try:- return get_secrets_provider().get_secret(secret_id=var_name)+ return secrets_provider.get_secret(secret_id=var_name)except SecretNotFoundError:continue # Try next optionelse:# No prefix, try as secrettry:- return get_secrets_provider().get_secret(secret_id=part)+ return secrets_provider.get_secret(secret_id=part)except SecretNotFoundError:continue # Try next optionmsg = f"Could not get variable from fallback pattern: {var_spec}"@@ -135,10 +136,10 @@ def _get_env_var(var_name: str) -> str:raise VarNotFoundError(message=msg, var_name=var_name) from exc-def _get_secret(secret_name: str) -> str:+def _get_secret(secret_name: str, secrets_provider: SecretsProviderAbstract) -> str:"""Get secret, raising VarNotFoundError if not found."""try:- return get_secrets_provider().get_secret(secret_id=secret_name)+ return secrets_provider.get_secret(secret_id=secret_name)except SecretNotFoundError as exc:msg = f"Could not get variable '{secret_name}': {exc!s}"raise VarNotFoundError(message=msg, var_name=secret_name) from excdiff --git a/pipelex/tools/typing/pydantic_utils.py b/pipelex/tools/typing/pydantic_utils.pyindex 9cc4f9705..a7688dd22 100644--- a/pipelex/tools/typing/pydantic_utils.py+++ b/pipelex/tools/typing/pydantic_utils.py@@ -45,12 +45,12 @@ def analyze_pydantic_validation_error(exc: ValidationError) -> PydanticValidatioerror_msg = "Validation error(s):"# Collect different types of validation errors- missing_fields = [f"{'.'.join(map(str, err['loc']))}" for err in exc.errors() if err["type"] == "missing"]- extra_fields = [f"{'.'.join(map(str, err['loc']))}: {err['input']}" for err in exc.errors() if err["type"] == "extra_forbidden"]- type_errors = [f"{'.'.join(map(str, err['loc']))}: expected {err['type']}" for err in exc.errors() if err["type"] == "type_error"]- value_errors = [f"{'.'.join(map(str, err['loc']))}: {err['msg']}" for err in exc.errors() if err["type"] == "value_error"]+ missing_fields = [f"'{'.'.join(map(str, err['loc']))}'" for err in exc.errors() if err["type"] == "missing"]+ extra_fields = [f"'{'.'.join(map(str, err['loc']))}'" for err in exc.errors() if err["type"] == "extra_forbidden"]+ type_errors = [f"'{'.'.join(map(str, err['loc']))}': expected {err['type']}" for err in exc.errors() if err["type"] == "type_error"]+ value_errors = [f"'{'.'.join(map(str, err['loc']))}': {err['msg']}" for err in exc.errors() if err["type"] == "value_error"]enum_errors = [- f"{'.'.join(map(str, err['loc']))}: invalid enum value '{err.get('input', 'unknown')}'" for err in exc.errors() if err["type"] == "enum"+ f"'{'.'.join(map(str, err['loc']))}': invalid enum value '{err.get('input', 'unknown')}'" for err in exc.errors() if err["type"] == "enum"]union_tag_errors: list[str] = []for err in exc.errors():@@ -58,7 +58,7 @@ def analyze_pydantic_validation_error(exc: ValidationError) -> PydanticValidatiofield_path = ".".join(map(str, err["loc"]))# Extract discriminator field name from contextdiscriminator = err.get("ctx", {}).get("discriminator", "type")- union_tag_errors.append(f"{field_path}: missing required discriminator field '{discriminator}'")+ union_tag_errors.append(f"'{field_path}': missing required discriminator field '{discriminator}'")model_type_errors: list[str] = []for err in exc.errors():@@ -68,23 +68,23 @@ def analyze_pydantic_validation_error(exc: ValidationError) -> PydanticValidatioexpected_type = err.get("ctx", {}).get("class_name", "unknown model type")actual_input = err.get("input", "unknown")actual_type = type(actual_input).__name__ if actual_input != "unknown" else "unknown"- model_type_errors.append(f"{field_path}: expected {expected_type}, got {actual_type}")+ model_type_errors.append(f"'{field_path}': expected {expected_type}, got {actual_type}")# Add each type of error to the message if presentif missing_fields:- error_msg += f"\n\nMissing required fields: {missing_fields}"+ error_msg += f"\n\nMissing required fields: {', '.join(missing_fields)}"if extra_fields:- error_msg += f"\n\nExtra forbidden fields: {extra_fields}"+ error_msg += f"\n\nExtra forbidden fields: {', '.join(extra_fields)}"if type_errors:- error_msg += f"\n\nType errors: {type_errors}"+ error_msg += f"\n\nType errors: {', '.join(type_errors)}"if value_errors:- error_msg += f"\n\nValue errors: {value_errors}"+ error_msg += f"\n\nValue errors: {', '.join(value_errors)}"if enum_errors:- error_msg += f"\n\nEnum errors: {enum_errors}"+ error_msg += f"\n\nEnum errors: {', '.join(enum_errors)}"if union_tag_errors:- error_msg += f"\n\nUnion discriminator errors: {union_tag_errors}"+ error_msg += f"\n\nUnion discriminator errors: {', '.join(union_tag_errors)}"if model_type_errors:- error_msg += f"\n\nModel type errors: {model_type_errors}"+ error_msg += f"\n\nModel type errors: {', '.join(model_type_errors)}"# If none of the specific error types were found, add the raw error messagesif not any([missing_fields, extra_fields, type_errors, value_errors, enum_errors, union_tag_errors, model_type_errors]):diff --git a/pipelex/urls.py b/pipelex/urls.pyindex 7d90529a1..67eef86f5 100644--- a/pipelex/urls.py+++ b/pipelex/urls.py@@ -5,3 +5,4 @@ class URLs:changelog = "https://docs.pipelex.com/changelog/"discord = "https://go.pipelex.com/discord"pipe_func_docs = "https://docs.pipelex.com/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc"+ backend_provider_docs = "https://docs.pipelex.com/pages/configuration/config-technical/inference-backend-config/"diff --git a/tests/acceptance/init_command.md b/tests/acceptance/init_command.mdnew file mode 100644index 000000000..ecc7d113c--- /dev/null+++ b/tests/acceptance/init_command.md@@ -0,0 +1,477 @@+# Init Command - Acceptance Test Checklist++This document provides a comprehensive checklist for manually testing all scenarios of the `pipelex init` command.++## Pre-Test Setup++Before each test scenario, ensure:++- [ ] You have a clean test environment (consider using a temporary directory)+- [ ] No `.pipelex/` directory exists in your test location+- [ ] You have write permissions in the test directory++## Test Scenarios++### 1. First-Time Full Initialization (`pipelex init`)++**Test Case 1.1: Complete initialization with default selections**++- [ ] Run: `pipelex init`+- [ ] Verify welcome panel is displayed with list of actions+- [ ] Press Enter to confirm initialization+- [ ] Verify config files are copied successfully+- [ ] Verify backend selection panel is displayed with numbered options+- [ ] Select option `1` (pipelex_inference) and press Enter+- [ ] Verify confirmation message shows 1 backend configured+- [ ] Verify routing profile is set to "pipelex_first"+- [ ] Verify telemetry selection panel is displayed+- [ ] Select option `1` (OFF) and press Enter+- [ ] Verify telemetry mode is set successfully+- [ ] Verify `.pipelex/` directory exists with all config files+- [ ] Verify `.pipelex/inference/backends.toml` has pipelex_inference enabled+- [ ] Verify `.pipelex/inference/routing_profiles.toml` has active = "pipelex_first"+- [ ] Verify `.pipelex/telemetry.toml` has telemetry_mode = "off"++**Test Case 1.2: Initialization with multiple backends**++- [ ] Run: `pipelex init`+- [ ] Confirm initialization+- [ ] Note the displayed backend numbers, then select 3 backends (e.g., openai, anthropic, mistral) using their indices+- [ ] Verify confirmation shows 3 backends configured+- [ ] Verify "Setting up routing for multiple backends" message+- [ ] Verify primary backend selection panel is displayed+- [ ] Select primary backend (e.g., `1` for the first in your selection)+- [ ] Verify fallback order panel is displayed+- [ ] Press Enter to keep default order or enter custom order (e.g., `2,1`)+- [ ] Verify "custom_routing" profile is created and set as active+- [ ] Verify `.pipelex/inference/routing_profiles.toml` contains custom_routing profile+- [ ] Verify fallback_order is set correctly++**Test Case 1.3: Initialization with all backends**++- [ ] Run: `pipelex init`+- [ ] Confirm initialization+- [ ] Enter `a` or `all` for backend selection+- [ ] Verify all backends (except internal) are enabled+- [ ] Follow through routing and telemetry setup+- [ ] Verify all backends are enabled in backends.toml++**Test Case 1.4: Cancel at backend selection**++- [ ] Run: `pipelex init`+- [ ] Confirm initialization+- [ ] Enter `q` or `quit` at backend selection+- [ ] Verify graceful exit with message+- [ ] Verify config files were created but backends.toml remains in template state++**Test Case 1.5: Cancel at initialization confirmation**++- [ ] Run: `pipelex init`+- [ ] Enter `n` when asked to continue+- [ ] Verify cancellation message+- [ ] Verify suggestion to run `pipelex init` later+- [ ] Verify no files were created++---++### 2. Focused Initialization - Config Only++**Test Case 2.1: Initialize config files only**++- [ ] Run: `pipelex init config`+- [ ] Confirm initialization+- [ ] Verify config files are copied+- [ ] Verify NO backend selection prompt appears+- [ ] Verify NO telemetry prompt appears+- [ ] Verify `.pipelex/` directory exists with config files+- [ ] Verify `.pipelex/inference/backends.toml` exists in template state++**Test Case 2.2: Config already initialized**++- [ ] Run: `pipelex init config` (after already initialized)+- [ ] Verify message: "Configuration files are already in place!"+- [ ] Verify tip about using `--reset`+- [ ] Verify no changes made++---++### 3. Focused Initialization - Inference Only++**Test Case 3.1: Initialize inference with existing config**++- [ ] Ensure `.pipelex/` exists (run `pipelex init config` first)+- [ ] Run: `pipelex init inference`+- [ ] Verify backend selection panel is displayed+- [ ] Note the displayed backend numbers, then select 2 backends (e.g., openai, anthropic) using their indices+- [ ] Verify backends are enabled in backends.toml+- [ ] Verify routing is configured automatically+- [ ] Verify NO telemetry prompt appears++**Test Case 3.2: Initialize inference without config files**++- [ ] Remove `.pipelex/` directory+- [ ] Run: `pipelex init inference`+- [ ] Verify warning about missing backends.toml+- [ ] Verify command handles gracefully++**Test Case 3.3: Inference already configured**++- [ ] Run: `pipelex init inference` (after already configured)+- [ ] Verify message: "Inference backends are already configured!"+- [ ] Verify question: "Would you like to reconfigure inference backends?"+- [ ] Enter `y` to reconfigure+- [ ] Verify backend selection panel appears with current selection shown+- [ ] Verify you can change backends+- [ ] Verify routing is updated based on new selection++**Test Case 3.4: Decline reconfiguration**++- [ ] Run: `pipelex init inference` (after already configured)+- [ ] Enter `n` when asked to reconfigure+- [ ] Verify "No changes made" message+- [ ] Verify no changes to backends.toml++---++### 4. Focused Initialization - Routing Only++**Test Case 4.1: Configure routing with multiple backends enabled**++- [ ] Ensure `.pipelex/` exists with multiple backends enabled+- [ ] Run: `pipelex init routing`+- [ ] Verify primary backend selection panel+- [ ] Select primary backend+- [ ] Verify fallback order configuration+- [ ] Configure order and confirm+- [ ] Verify routing_profiles.toml is updated with custom_routing++**Test Case 4.2: Configure routing with single backend**++- [ ] Ensure only one backend is enabled+- [ ] Run: `pipelex init routing`+- [ ] Verify profile is set to `all_{backend_key}`+- [ ] Verify no primary/fallback prompts++**Test Case 4.3: Configure routing with no backends enabled**++- [ ] Disable all backends in backends.toml+- [ ] Run: `pipelex init routing`+- [ ] Verify warning: "No backends enabled. Please run 'pipelex init inference' first."+- [ ] Verify no changes to routing_profiles.toml++**Test Case 4.4: Routing already configured**++- [ ] Run: `pipelex init routing` (after already configured)+- [ ] Verify message: "Routing profile is already configured!"+- [ ] Verify question about reconfiguration+- [ ] Test both accepting and declining reconfiguration++---++### 5. Focused Initialization - Telemetry Only++**Test Case 5.1: Initialize telemetry only**++- [ ] Ensure `.pipelex/` exists without telemetry.toml+- [ ] Run: `pipelex init telemetry`+- [ ] Verify telemetry selection panel+- [ ] Select option `2` (ANONYMOUS)+- [ ] Verify telemetry.toml is created with mode = "anonymous"+- [ ] Verify NO backend or routing prompts++**Test Case 5.2: Telemetry already configured**++- [ ] Run: `pipelex init telemetry` (after already configured)+- [ ] Verify message: "Telemetry preferences are already configured!"+- [ ] Verify question about reconfiguration+- [ ] Enter `y` to reconfigure+- [ ] Select different mode+- [ ] Verify telemetry.toml is updated++**Test Case 5.3: Cancel telemetry selection**++- [ ] Run: `pipelex init telemetry`+- [ ] Enter `q` at telemetry prompt+- [ ] Verify graceful exit+- [ ] Verify telemetry.toml is not created++---++### 6. Reset Operations++**Test Case 6.1: Reset all with --reset**++- [ ] Ensure `.pipelex/` fully configured+- [ ] Run: `pipelex init --reset`+- [ ] Verify panel shows "Resetting Configuration" (yellow)+- [ ] Verify all items listed for reset+- [ ] Confirm reset+- [ ] Go through all selections again+- [ ] Verify all files are overwritten+- [ ] Verify new selections are applied++**Test Case 6.2: Reset config only**++- [ ] Run: `pipelex init config --reset`+- [ ] Verify config files are overwritten+- [ ] Verify backend selection prompt appears+- [ ] Complete configuration+- [ ] Verify backends.toml and routing are updated++**Test Case 6.3: Reset inference only**++- [ ] Run: `pipelex init inference --reset`+- [ ] Verify backend selection appears+- [ ] Change backend selection+- [ ] Verify backends.toml is updated+- [ ] Verify routing is reconfigured++**Test Case 6.4: Reset routing only**++- [ ] Run: `pipelex init routing --reset`+- [ ] Verify routing configuration prompts+- [ ] Change routing configuration+- [ ] Verify routing_profiles.toml is updated++**Test Case 6.5: Reset telemetry only**++- [ ] Run: `pipelex init telemetry --reset`+- [ ] Verify telemetry prompt+- [ ] Change telemetry mode+- [ ] Verify telemetry.toml is updated++---++### 7. All Already Configured++**Test Case 7.1: Everything configured**++- [ ] Ensure complete initialization+- [ ] Run: `pipelex init`+- [ ] Verify message: "Pipelex is already fully initialized!"+- [ ] Verify tip about `--reset`+- [ ] Verify no prompts appear+- [ ] Verify no changes made++---++### 8. Input Validation++**Test Case 8.1: Invalid backend selection - out of range**++- [ ] Run: `pipelex init`+- [ ] Enter invalid index like `99`+- [ ] Verify error message with valid range+- [ ] Verify prompt repeats+- [ ] Enter valid selection+- [ ] Verify successful continuation++**Test Case 8.2: Invalid backend selection - non-numeric**++- [ ] Run: `pipelex init`+- [ ] Enter invalid input like `abc`+- [ ] Verify error message+- [ ] Verify prompt repeats+- [ ] Enter valid selection++**Test Case 8.3: Space-separated backend indices**++- [ ] Run: `pipelex init`+- [ ] Note the displayed backend numbers, then enter 3 backend indices with spaces (e.g., for openai, anthropic, mistral)+- [ ] Verify backends are parsed correctly+- [ ] Verify all selected backends are enabled++**Test Case 8.4: Default selection (press Enter)**++- [ ] Run: `pipelex init`+- [ ] Press Enter at backend selection (accept default)+- [ ] Verify default backend (1 - pipelex_inference) is selected++**Test Case 8.5: Keep current selection**++- [ ] Run: `pipelex init inference` (with existing selection)+- [ ] Agree to reconfigure+- [ ] Press Enter at backend selection+- [ ] Verify current selection is kept++**Test Case 8.6: Invalid telemetry selection**++- [ ] Run: `pipelex init`+- [ ] Complete to telemetry prompt+- [ ] Enter invalid option like `5`+- [ ] Verify error message+- [ ] Enter valid option+- [ ] Verify successful completion++**Test Case 8.7: Invalid fallback order - wrong count**++- [ ] Select 3+ backends+- [ ] At fallback order prompt, enter only 2 indices+- [ ] Verify error: "You must specify all X backends"+- [ ] Enter correct count of indices+- [ ] Verify successful ordering++**Test Case 8.8: Invalid fallback order - duplicates**++- [ ] Select 3+ backends+- [ ] At fallback order prompt, enter `1 1 2`+- [ ] Verify error: "Duplicate indices not allowed"+- [ ] Enter valid unique indices+- [ ] Verify successful ordering++---++### 9. Edge Cases++**Test Case 9.1: pipelex_inference always sets pipelex_first**++- [ ] Run: `pipelex init`+- [ ] Select pipelex_inference as one of multiple backends+- [ ] Verify routing is set to "pipelex_first" automatically+- [ ] Verify NO primary/fallback prompts++**Test Case 9.2: Single non-pipelex backend**++- [ ] Run: `pipelex init`+- [ ] Note the displayed backend numbers, then select only one backend (e.g., openai) using its index+- [ ] Verify routing is set to "all_{backend_key}" (e.g., "all_openai")+- [ ] Verify profile is created if it doesn't exist+- [ ] Verify NO primary/fallback prompts++**Test Case 9.3: Profile creation confirmation**++- [ ] Select a single obscure backend+- [ ] If profile doesn't exist in template, verify prompt to create+- [ ] Accept profile creation+- [ ] Verify profile is created in routing_profiles.toml++**Test Case 9.4: Decline profile creation**++- [ ] Select a backend without existing profile+- [ ] Decline profile creation when prompted+- [ ] Verify message: "Keeping current routing profile configuration"+- [ ] Verify no changes to routing_profiles.toml++**Test Case 9.5: Two backends (automatic fallback)**++- [ ] Select exactly 2 backends+- [ ] Select primary backend+- [ ] Verify fallback_order is set automatically with both backends+- [ ] Verify NO fallback ordering prompt (only 1 remaining)++---++### 10. Files and Directory Structure++**Test Case 10.1: Verify directory structure**++After complete initialization, verify:++- [ ] `.pipelex/` directory exists+- [ ] `.pipelex/pipelex.toml` exists+- [ ] `.pipelex/inference/` directory exists+- [ ] `.pipelex/inference/backends.toml` exists+- [ ] `.pipelex/inference/routing_profiles.toml` exists+- [ ] `.pipelex/telemetry.toml` exists+- [ ] All other expected config files from template are present++**Test Case 10.2: Verify file contents - backends.toml**++- [ ] Open `.pipelex/inference/backends.toml`+- [ ] Verify selected backends have `enabled = true`+- [ ] Verify non-selected backends have `enabled = false`+- [ ] Verify `internal` backend is always enabled++**Test Case 10.3: Verify file contents - routing_profiles.toml**++- [ ] Open `.pipelex/inference/routing_profiles.toml`+- [ ] Verify `active` field matches expected profile+- [ ] If custom_routing: verify `default` and `fallback_order` fields+- [ ] Verify `routes` section exists++**Test Case 10.4: Verify file contents - telemetry.toml**++- [ ] Open `.pipelex/telemetry.toml`+- [ ] Verify `telemetry_mode` matches selection+- [ ] Verify valid values: "off", "anonymous", or "identified"++**Test Case 10.5: Existing files not overwritten (without reset)**++- [ ] Modify `.pipelex/pipelex.toml` (add a comment)+- [ ] Run: `pipelex init config`+- [ ] Verify your comment is preserved+- [ ] Verify message about skipped existing files++---++### 11. Doctor Integration++**Test Case 11.1: Doctor command suggests init**++- [ ] Remove `.pipelex/` directory+- [ ] Run: `pipelex doctor`+- [ ] Verify doctor detects missing configuration+- [ ] Follow suggestion to run init+- [ ] Verify successful configuration++---++### 12. Performance and UX++**Test Case 12.1: Responsive feedback**++- [ ] Run: `pipelex init`+- [ ] Verify all prompts appear immediately+- [ ] Verify success messages appear after actions+- [ ] Verify progress is clear at each step++**Test Case 12.2: Clear instructions**++- [ ] Run through full initialization+- [ ] Verify all panels have clear titles+- [ ] Verify all prompts have clear instructions+- [ ] Verify examples are provided (e.g., "1,5,6 or 1 5 6")+- [ ] Verify default values are clearly indicated++**Test Case 12.3: Error recovery**++- [ ] Make intentional errors at each prompt+- [ ] Verify error messages are helpful+- [ ] Verify you can recover and continue+- [ ] Verify no need to restart from beginning++---++## Test Environment Cleanup++After completing tests:++- [ ] Remove test `.pipelex/` directories+- [ ] Document any issues found+- [ ] Note any unclear UX or confusing prompts+- [ ] Record any unexpected behavior++---++## Sign-Off++**Tester Name:** _________________________++**Date:** _________________________++**Test Environment:**++- OS: _________________________+- Python Version: _________________________+- Pipelex Version: _________________________++**Overall Result:** [ ] PASS [ ] FAIL++**Notes:**++_____________________________________________________________________________++_____________________________________________________________________________++_____________________________________________________________________________+diff --git a/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py b/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.pynew file mode 100644index 000000000..2cefa1f4b--- /dev/null+++ b/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py@@ -0,0 +1,160 @@+from __future__ import annotations++from pathlib import Path+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from pytest_mock import MockerFixture++from pipelex.cli.commands.init.command import init_cmd+from pipelex.cli.commands.init.ui.types import InitFocus+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper+++class TestEdgeCases:+ def test_pipelex_inference_sets_pipelex_first(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 9.1: pipelex_inference always sets pipelex_first."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for pipelex_inference and openai+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["pipelex_inference", "openai"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs - no primary/fallback prompts expected+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select pipelex_inference and openai+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify pipelex_first is set automatically+ env.verify_routing("pipelex_first")++ def test_single_non_pipelex_backend(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 9.2: Single non-pipelex backend."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get index for openai only+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["openai"])++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(str(indices[0])) # Select only openai+ env.add_confirm_input(True) # Confirm creating profile if needed+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify routing is set to all_openai+ env.verify_routing("all_openai")++ def test_two_backends_automatic_fallback(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 9.5: Two backends (automatic fallback)."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for exactly 2 backends (anthropic first, then openai)+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "openai"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs - no fallback order prompt for exactly 2 backends+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select anthropic, openai+ env.add_prompt_input("1") # Primary backend: anthropic (first in selection)+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify custom routing with automatic fallback+ env.verify_routing("custom_routing", expected_default="anthropic", expected_fallback_order=["anthropic", "openai"])++ def test_reset_all_with_flag(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 6.1: Reset all with --reset flag."""+ # Setup environment with existing config+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # Set initial state+ backends_path = env.inference_dir / "backends.toml"+ toml_doc = load_toml_with_tomlkit(str(backends_path))+ toml_doc["pipelex_inference"]["enabled"] = True # type: ignore[index]+ save_toml_to_path(toml_doc, str(backends_path))++ telemetry_path = env.pipelex_dir / "telemetry.toml"+ toml_doc_tel = load_toml_with_tomlkit(str(telemetry_path))+ toml_doc_tel["telemetry_mode"] = "identified"+ save_toml_to_path(toml_doc_tel, str(telemetry_path))++ # Get index for anthropic+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic"])++ # User inputs+ env.add_confirm_input(True) # Confirm reset+ env.add_prompt_input(str(indices[0])) # Select anthropic+ env.add_confirm_input(True) # Confirm creating profile if needed+ env.add_prompt_input("1") # Telemetry: OFF++ env.setup_mocks()++ # Execute with reset flag+ init_cmd(focus=InitFocus.ALL, reset=True)++ # Verify configuration was reset+ env.verify_backends_enabled(["anthropic"])+ env.verify_telemetry("off")++ def test_verify_backends_toml_contents(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 10.2: Verify backends.toml contents."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for specific backends+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "mistral"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select openai, mistral+ env.add_prompt_input("1") # Primary backend+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify detailed backends.toml contents+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))++ # Selected backends should be enabled+ assert toml_doc["openai"]["enabled"] is True # type: ignore[index]+ assert toml_doc["mistral"]["enabled"] is True # type: ignore[index]++ # Non-selected backends should be disabled+ assert toml_doc["pipelex_inference"]["enabled"] is False # type: ignore[index]+ assert toml_doc["anthropic"]["enabled"] is False # type: ignore[index]++ # Internal backend should be enabled+ assert toml_doc["internal"]["enabled"] is True # type: ignore[index]diff --git a/tests/e2e/pipelex/cli/test_init_cmd_first_time.py b/tests/e2e/pipelex/cli/test_init_cmd_first_time.pynew file mode 100644index 000000000..d92589109--- /dev/null+++ b/tests/e2e/pipelex/cli/test_init_cmd_first_time.py@@ -0,0 +1,145 @@+from __future__ import annotations++from pathlib import Path+from typing import TYPE_CHECKING++import pytest+import typer++if TYPE_CHECKING:+ from pytest_mock import MockerFixture++from pipelex.cli.commands.init.command import init_cmd+from pipelex.cli.commands.init.ui.types import InitFocus+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper+++class TestFirstTimeInitialization:+ def test_complete_init_with_default_selections(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 1.1: Complete initialization with default selections."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs: confirm, default backend (1), telemetry OFF (1)+ env.add_confirm_input(True)+ env.add_prompt_input("1") # Default: pipelex_inference+ env.add_prompt_input("1") # Telemetry: OFF++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify+ env.verify_file_exists("pipelex.toml")+ env.verify_file_exists("inference/backends.toml")+ env.verify_file_exists("inference/routing_profiles.toml")+ env.verify_file_exists("telemetry.toml")+ env.verify_backends_enabled(["pipelex_inference"])+ env.verify_routing("pipelex_first")+ env.verify_telemetry("off")++ def test_init_with_multiple_backends_and_routing(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 1.2: Initialization with multiple backends."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for anthropic, mistral, openai (in this order for testing)+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "mistral", "openai"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select 3 backends+ env.add_prompt_input("1") # Primary: first one (anthropic)+ env.add_prompt_input("2,1") # Custom fallback order (mistral, anthropic)+ env.add_prompt_input("2") # Telemetry: ANONYMOUS++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify backends+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])++ # Verify custom routing+ env.verify_routing("custom_routing", expected_default="anthropic")++ # Verify telemetry+ env.verify_telemetry("anonymous")++ def test_init_with_all_backends(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 1.3: Initialization with all backends."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("all") # Select all backends+ env.add_prompt_input("1") # Telemetry: OFF++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify all backends are enabled+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))+ for backend_key in toml_doc:+ if backend_key != "internal":+ assert toml_doc[backend_key]["enabled"] is True # type: ignore[index]++ # Verify routing (pipelex_first since pipelex_inference is included)+ env.verify_routing("pipelex_first")++ def test_cancel_at_backend_selection(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 1.4: Cancel at backend selection."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs: confirm, then quit at backend selection+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("q") # Quit at backend selection++ env.setup_mocks()++ # Execute - may raise an exit exception on cancellation+ try:+ init_cmd(focus=InitFocus.ALL, reset=False)+ except (typer.Exit, SystemExit):+ # Expected: user quit at backend selection+ pass++ # Verify config files were created but backends remain in template state+ env.verify_file_exists("inference/backends.toml")++ # Verify pipelex_inference is still enabled (default template state)+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))+ assert toml_doc["pipelex_inference"]["enabled"] is True # type: ignore[index]++ def test_cancel_at_initialization_confirmation(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 1.5: Cancel at initialization confirmation."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs: decline confirmation+ env.add_confirm_input(False) # Decline initialization++ env.setup_mocks()++ # Execute - should raise typer.Exit+ with pytest.raises(typer.Exit):+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify no files were created+ env.verify_file_not_exists("pipelex.toml")+ env.verify_file_not_exists("inference/backends.toml")diff --git a/tests/e2e/pipelex/cli/test_init_cmd_focused.py b/tests/e2e/pipelex/cli/test_init_cmd_focused.pynew file mode 100644index 000000000..a4e1aac3a--- /dev/null+++ b/tests/e2e/pipelex/cli/test_init_cmd_focused.py@@ -0,0 +1,211 @@+from __future__ import annotations++from pathlib import Path+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from pytest_mock import MockerFixture++from pipelex.cli.commands.init.command import init_cmd+from pipelex.cli.commands.init.ui.types import InitFocus+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper+++class TestFocusedInitialization:+ def test_config_only_initialization(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 2.1: Initialize config files only."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("1") # Backend selection+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute with CONFIG focus+ init_cmd(focus=InitFocus.CONFIG, reset=False)++ # Verify config files exist+ env.verify_file_exists("pipelex.toml")+ env.verify_file_exists("inference/backends.toml")++ def test_inference_with_existing_config(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 3.1: Initialize inference with existing config."""+ # Setup environment with existing config+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)++ # Get indices for anthropic and openai+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "openai"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs - need to confirm reconfigure since backends already exist+ env.add_confirm_input(True) # Confirm reconfigure+ env.add_prompt_input(indices_str) # Select 2 backends+ env.add_prompt_input("1") # Primary backend (anthropic - first in selection)++ env.setup_mocks()++ # Execute with INFERENCE focus+ init_cmd(focus=InitFocus.INFERENCE, reset=False)++ # Verify backends are enabled+ env.verify_backends_enabled(["openai", "anthropic"])++ # Verify routing is configured+ env.verify_routing("custom_routing", expected_default="anthropic")++ # Verify NO telemetry prompt appeared+ env.verify_file_not_exists("telemetry.toml")++ def test_reconfigure_inference(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 3.3: Inference already configured - reconfigure."""+ # Setup environment with existing config+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # Set pipelex_inference as initially enabled+ backends_path = env.inference_dir / "backends.toml"+ toml_doc = load_toml_with_tomlkit(str(backends_path))+ toml_doc["pipelex_inference"]["enabled"] = True # type: ignore[index]+ save_toml_to_path(toml_doc, str(backends_path))++ # Get index for mistral+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["mistral"])++ # User inputs+ env.add_confirm_input(True) # Confirm reconfigure+ env.add_prompt_input(str(indices[0])) # Change to mistral+ env.add_confirm_input(True) # Confirm creating profile if needed++ env.setup_mocks()++ # Execute with INFERENCE focus+ init_cmd(focus=InitFocus.INFERENCE, reset=False)++ # Verify backend was changed+ env.verify_backends_enabled(["mistral"])++ # Verify routing was updated+ env.verify_routing("all_mistral")++ def test_configure_routing_with_multiple_backends(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 4.1: Configure routing with multiple backends enabled."""+ # Setup environment with existing config and multiple backends enabled+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # Enable multiple backends (anthropic, mistral, openai - in TOML order)+ # Disable pipelex_inference so we get custom_routing instead of pipelex_first+ backends_path = env.inference_dir / "backends.toml"+ toml_doc = load_toml_with_tomlkit(str(backends_path))+ toml_doc["pipelex_inference"]["enabled"] = False # type: ignore[index]+ toml_doc["anthropic"]["enabled"] = True # type: ignore[index]+ toml_doc["mistral"]["enabled"] = True # type: ignore[index]+ toml_doc["openai"]["enabled"] = True # type: ignore[index]+ save_toml_to_path(toml_doc, str(backends_path))++ # User inputs for routing - need to confirm reconfigure since routing already exists+ env.add_confirm_input(True) # Confirm reconfigure+ # The backends are enabled and will be listed as: anthropic, mistral, openai+ env.add_prompt_input("1") # Primary backend: first one (anthropic)+ env.add_prompt_input("1,2") # Fallback order for remaining 2++ env.setup_mocks()++ # Execute with ROUTING focus+ init_cmd(focus=InitFocus.ROUTING, reset=False)++ # Verify routing was configured+ env.verify_routing("custom_routing", expected_default="anthropic")++ def test_configure_routing_with_single_backend(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 4.2: Configure routing with single backend."""+ # Setup environment with existing config and single backend enabled+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # Enable only openai+ backends_path = env.inference_dir / "backends.toml"+ toml_doc = load_toml_with_tomlkit(str(backends_path))+ toml_doc["openai"]["enabled"] = True # type: ignore[index]+ toml_doc["pipelex_inference"]["enabled"] = False # type: ignore[index]+ save_toml_to_path(toml_doc, str(backends_path))++ # User inputs+ env.add_confirm_input(True) # Confirm creating profile if needed++ env.setup_mocks()++ # Execute with ROUTING focus+ init_cmd(focus=InitFocus.ROUTING, reset=False)++ # Verify routing is set to all_openai+ env.verify_routing("all_openai")++ def test_telemetry_only_initialization(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 5.1: Initialize telemetry only."""+ # Setup environment with existing config but no telemetry+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("2") # Telemetry: ANONYMOUS++ env.setup_mocks()++ # Execute with TELEMETRY focus+ init_cmd(focus=InitFocus.TELEMETRY, reset=False)++ # Verify telemetry was created+ env.verify_telemetry("anonymous")++ def test_reconfigure_telemetry(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 5.2: Telemetry already configured - reconfigure."""+ # Setup environment with existing telemetry+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # Set initial telemetry to OFF+ telemetry_path = env.pipelex_dir / "telemetry.toml"+ toml_doc = load_toml_with_tomlkit(str(telemetry_path))+ toml_doc["telemetry_mode"] = "off"+ save_toml_to_path(toml_doc, str(telemetry_path))++ # User inputs+ env.add_confirm_input(True) # Confirm reconfigure+ env.add_prompt_input("3") # Change to IDENTIFIED++ env.setup_mocks()++ # Execute with TELEMETRY focus+ init_cmd(focus=InitFocus.TELEMETRY, reset=False)++ # Verify telemetry was changed+ env.verify_telemetry("identified")++ def test_everything_already_configured(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 7.1: Everything configured - decline reconfigure."""+ # Setup complete environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # User inputs: decline reconfigure - no prompts needed if everything is configured+ # The command should detect everything is configured and exit++ env.setup_mocks()++ # Execute - should complete immediately since everything is configured+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Should complete without errors (no changes made)+ # Verify files still exist+ env.verify_file_exists("telemetry.toml")diff --git a/tests/e2e/pipelex/cli/test_init_cmd_validation.py b/tests/e2e/pipelex/cli/test_init_cmd_validation.pynew file mode 100644index 000000000..1956bf77e--- /dev/null+++ b/tests/e2e/pipelex/cli/test_init_cmd_validation.py@@ -0,0 +1,173 @@+from __future__ import annotations++from pathlib import Path+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from pytest_mock import MockerFixture++from pipelex.cli.commands.init.command import init_cmd+from pipelex.cli.commands.init.ui.types import InitFocus+from pipelex.kit.paths import get_kit_configs_dir+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper+++class TestInputValidation:+ def test_invalid_backend_index_then_valid(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 8.1: Invalid backend index, then valid."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs: invalid index, then valid+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("99") # Invalid index+ env.add_prompt_input("1") # Valid: pipelex_inference+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify successful completion with valid selection+ env.verify_backends_enabled(["pipelex_inference"])++ def test_invalid_non_numeric_input_then_valid(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 8.2: Non-numeric input, then valid."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("abc") # Invalid non-numeric+ env.add_prompt_input("1") # Valid: pipelex_inference+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify successful completion+ env.verify_backends_enabled(["pipelex_inference"])++ def test_space_separated_backend_indices(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 8.3: Space-separated backend indices."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for openai, anthropic, mistral+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "anthropic", "mistral"])+ indices_str = " ".join(str(i) for i in indices) # Space-separated++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Space-separated indices+ env.add_prompt_input("1") # Primary backend+ env.add_prompt_input("") # Accept default fallback order+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify all backends are enabled+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])++ def test_default_selection_press_enter(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 8.4: Default selection (press Enter)."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs: empty string for default+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("") # Empty = default (pipelex_inference)+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify default backend is selected+ env.verify_backends_enabled(["pipelex_inference"])++ def test_invalid_telemetry_selection_then_valid(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 8.6: Invalid telemetry selection."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("1") # Backend selection+ env.add_prompt_input("5") # Invalid telemetry option+ env.add_prompt_input("2") # Valid: ANONYMOUS++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify successful completion with valid telemetry+ env.verify_telemetry("anonymous")++ def test_invalid_fallback_order_wrong_count(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 8.7: Invalid fallback order - wrong count."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for 3 backends+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "anthropic", "mistral"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select 3 backends+ env.add_prompt_input("1") # Primary backend+ env.add_prompt_input("1") # Invalid: only 1 index instead of 2+ env.add_prompt_input("1,2") # Valid: 2 indices for remaining backends+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify successful completion+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])++ def test_invalid_fallback_order_duplicates(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test Case 8.8: Invalid fallback order - duplicates."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for 3 backends+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["openai", "anthropic", "mistral"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select 3 backends+ env.add_prompt_input("1") # Primary backend+ env.add_prompt_input("1,1") # Invalid: duplicate indices+ env.add_prompt_input("1,2") # Valid: unique indices+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify successful completion+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.pydeleted file mode 100644index 4135a9df5..000000000--- a/tests/helpers/__init__.py+++ /dev/null@@ -1 +0,0 @@-"""Test helper utilities package."""diff --git a/tests/helpers/init_cmd_helpers.py b/tests/helpers/init_cmd_helpers.pynew file mode 100644index 000000000..1eaf38e40--- /dev/null+++ b/tests/helpers/init_cmd_helpers.py@@ -0,0 +1,316 @@+from __future__ import annotations++import shutil+from pathlib import Path+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from pytest_mock import MockerFixture++from pipelex.kit.paths import get_kit_configs_dir+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit+++def get_backend_indices_helper(backends_toml_path: str, backend_names: list[str]) -> list[int]:+ """Get 1-based indices for given backend names from backends.toml.++ Args:+ backends_toml_path: Path to backends.toml file.+ backend_names: List of backend keys to find indices for.++ Returns:+ List of 1-based indices corresponding to the backend names.+ """+ toml_doc = load_toml_with_tomlkit(backends_toml_path)+ backend_list: list[str] = [key for key in toml_doc if key != "internal"]+ indices: list[int] = []+ for name in backend_names:+ if name in backend_list:+ # 1-based index for user input+ indices.append(backend_list.index(name) + 1)+ return indices+++def verify_backends_toml(backends_toml_path: str, expected_enabled: list[str]) -> None:+ """Verify backends.toml has expected backends enabled.++ Args:+ backends_toml_path: Path to backends.toml file.+ expected_enabled: List of backend keys expected to be enabled.+ """+ toml_doc = load_toml_with_tomlkit(backends_toml_path)++ for backend_key in toml_doc:+ if backend_key == "internal":+ # Internal should always be enabled+ assert toml_doc[backend_key]["enabled"] is True # type: ignore[index]+ elif backend_key in expected_enabled:+ assert toml_doc[backend_key]["enabled"] is True, f"Expected {backend_key} to be enabled" # type: ignore[index]+ # Check if enabled field exists before asserting+ elif "enabled" in toml_doc[backend_key]: # type: ignore[operator]+ assert toml_doc[backend_key]["enabled"] is False, f"Expected {backend_key} to be disabled" # type: ignore[index]+++def verify_routing_profile(+ routing_profiles_toml_path: str,+ expected_active: str,+ expected_default: str | None = None,+ expected_fallback_order: list[str] | None = None,+) -> None:+ """Verify routing_profiles.toml has expected configuration.++ Args:+ routing_profiles_toml_path: Path to routing_profiles.toml file.+ expected_active: Expected active profile name.+ expected_default: Expected default backend (optional).+ expected_fallback_order: Expected fallback order (optional).+ """+ toml_doc = load_toml_with_tomlkit(routing_profiles_toml_path)++ assert toml_doc["active"] == expected_active, f"Expected active profile '{expected_active}', got '{toml_doc['active']}'"++ if expected_default is not None or expected_fallback_order is not None:+ profiles: dict[str, Any] = toml_doc.get("profiles", {}) # type: ignore[assignment]+ assert expected_active in profiles, f"Profile '{expected_active}' not found"++ profile = profiles[expected_active] # type: ignore[index]++ if expected_default is not None:+ assert profile.get("default") == expected_default, f"Expected default '{expected_default}'" # type: ignore[union-attr]++ if expected_fallback_order is not None:+ actual_fallback = profile.get("fallback_order") # type: ignore[union-attr]+ assert actual_fallback == expected_fallback_order, f"Expected fallback_order {expected_fallback_order}, got {actual_fallback}"+++def verify_telemetry_config(telemetry_config_path: str, expected_mode: str) -> None:+ """Verify telemetry.toml has expected mode.++ Args:+ telemetry_config_path: Path to telemetry.toml file.+ expected_mode: Expected telemetry mode ("off", "anonymous", or "identified").+ """+ toml_doc = load_toml_with_tomlkit(telemetry_config_path)+ assert toml_doc["telemetry_mode"] == expected_mode, f"Expected telemetry_mode '{expected_mode}', got '{toml_doc['telemetry_mode']}'"+++def setup_pipelex_dir(tmp_path: Path, copy_configs: bool = True, copy_backends: bool = True, copy_routing: bool = True) -> Path:+ """Set up a test .pipelex directory structure.++ Args:+ tmp_path: Pytest temporary path fixture.+ copy_configs: Whether to copy config files from kit.+ copy_backends: Whether to copy backends.toml.+ copy_routing: Whether to copy routing_profiles.toml.++ Returns:+ Path to created .pipelex directory.+ """+ pipelex_dir = tmp_path / ".pipelex"+ pipelex_dir.mkdir(exist_ok=True)++ inference_dir = pipelex_dir / "inference"+ inference_dir.mkdir(exist_ok=True)++ kit_configs_dir = Path(str(get_kit_configs_dir()))++ if copy_configs:+ # Copy main pipelex.toml+ if (kit_configs_dir / "pipelex.toml").exists():+ shutil.copy2(kit_configs_dir / "pipelex.toml", pipelex_dir / "pipelex.toml")++ if copy_backends:+ # Copy backends.toml+ backends_source = kit_configs_dir / "inference" / "backends.toml"+ if backends_source.exists():+ shutil.copy2(backends_source, inference_dir / "backends.toml")++ if copy_routing:+ # Copy routing_profiles.toml+ routing_source = kit_configs_dir / "inference" / "routing_profiles.toml"+ if routing_source.exists():+ shutil.copy2(routing_source, inference_dir / "routing_profiles.toml")++ return pipelex_dir+++class MockedInitEnvironment:+ """Encapsulates all mocking setup for init command testing.++ This class manages:+ - File system setup (tmp_path, .pipelex directory)+ - Mocking of Console, Prompt, Confirm+ - User input simulation+ - Configuration verification+ """++ def __init__(self, tmp_path: Path, mocker: MockerFixture):+ """Initialize mocked environment.++ Args:+ tmp_path: Pytest temporary path fixture.+ mocker: Pytest mocker fixture.+ """+ self.tmp_path = tmp_path+ self.mocker = mocker+ self.pipelex_dir = tmp_path / ".pipelex"+ self.inference_dir = self.pipelex_dir / "inference"++ # Store mocks for later use+ self.mock_console: Any = None+ self.mock_prompt_ask: Any = None+ self.mock_confirm_ask: Any = None+ self.mock_config_manager: Any = None++ # Input sequences+ self.prompt_inputs: list[str] = []+ self.confirm_inputs: list[bool] = []++ def setup_empty_dir(self) -> None:+ """Set up an empty .pipelex directory structure."""+ self.pipelex_dir.mkdir(exist_ok=True)+ self.inference_dir.mkdir(exist_ok=True)++ def setup_with_configs(self, include_backends: bool = True, include_routing: bool = True, include_telemetry: bool = False) -> None:+ """Set up .pipelex directory with config files from kit.++ Args:+ include_backends: Whether to include backends.toml.+ include_routing: Whether to include routing_profiles.toml.+ include_telemetry: Whether to include telemetry.toml.+ """+ setup_pipelex_dir(+ self.tmp_path,+ copy_configs=True,+ copy_backends=include_backends,+ copy_routing=include_routing,+ )++ if include_telemetry:+ kit_configs_dir = Path(str(get_kit_configs_dir()))+ telemetry_source = kit_configs_dir / "telemetry.toml"+ if telemetry_source.exists():+ shutil.copy2(telemetry_source, self.pipelex_dir / "telemetry.toml")++ def mock_config_manager_paths(self) -> None:+ """Mock config_manager to use tmp_path."""+ self.mock_config_manager = self.mocker.MagicMock()+ self.mock_config_manager.pipelex_config_dir = str(self.pipelex_dir)++ # Patch all locations where config_manager is used+ self.mocker.patch("pipelex.cli.commands.init.command.config_manager", self.mock_config_manager)+ self.mocker.patch("pipelex.cli.commands.init.backends.config_manager", self.mock_config_manager)+ self.mocker.patch("pipelex.cli.commands.init.routing.config_manager", self.mock_config_manager)+ self.mocker.patch("pipelex.cli.commands.init.config_files.config_manager", self.mock_config_manager)++ def mock_console_outputs(self) -> None:+ """Mock Console to suppress output."""+ self.mock_console = self.mocker.patch("pipelex.cli.commands.init.command.Console")+ self.mocker.patch("pipelex.cli.commands.init.backends.Console")+ self.mocker.patch("pipelex.cli.commands.init.routing.Console")+ self.mocker.patch("pipelex.cli.commands.init.telemetry.Console")+ self.mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Console")+ self.mocker.patch("pipelex.cli.commands.init.ui.routing_ui.Console")+ self.mocker.patch("pipelex.cli.commands.init.ui.telemetry_ui.Console")++ def add_prompt_input(self, value: str) -> None:+ """Add a prompt input to the sequence.++ Args:+ value: The input value to simulate.+ """+ self.prompt_inputs.append(value)++ def add_confirm_input(self, value: bool) -> None:+ """Add a confirm input to the sequence.++ Args:+ value: The boolean confirmation to simulate.+ """+ self.confirm_inputs.append(value)++ def setup_mocks(self) -> None:+ """Set up all mocks with configured inputs."""+ self.mock_config_manager_paths()+ self.mock_console_outputs()++ # Mock Prompt.ask with side_effect for sequential inputs+ if self.prompt_inputs:+ self.mock_prompt_ask = self.mocker.patch(+ "rich.prompt.Prompt.ask",+ side_effect=self.prompt_inputs,+ )++ # Mock Confirm.ask with side_effect for sequential inputs+ if self.confirm_inputs:+ self.mock_confirm_ask = self.mocker.patch(+ "rich.prompt.Confirm.ask",+ side_effect=self.confirm_inputs,+ )++ # Mock typer.echo to suppress output+ self.mocker.patch("typer.echo")++ def get_backend_indices(self, backend_names: list[str]) -> list[int]:+ """Get 1-based indices for backend names.++ Args:+ backend_names: List of backend keys.++ Returns:+ List of 1-based indices.+ """+ backends_path = str(self.inference_dir / "backends.toml")+ return get_backend_indices_helper(backends_path, backend_names)++ def verify_backends_enabled(self, expected_enabled: list[str]) -> None:+ """Verify backends.toml has expected backends enabled.++ Args:+ expected_enabled: List of backend keys expected to be enabled.+ """+ backends_path = str(self.inference_dir / "backends.toml")+ verify_backends_toml(backends_path, expected_enabled)++ def verify_routing(+ self,+ expected_active: str,+ expected_default: str | None = None,+ expected_fallback_order: list[str] | None = None,+ ) -> None:+ """Verify routing profile configuration.++ Args:+ expected_active: Expected active profile name.+ expected_default: Expected default backend (optional).+ expected_fallback_order: Expected fallback order (optional).+ """+ routing_path = str(self.inference_dir / "routing_profiles.toml")+ verify_routing_profile(routing_path, expected_active, expected_default, expected_fallback_order)++ def verify_telemetry(self, expected_mode: str) -> None:+ """Verify telemetry configuration.++ Args:+ expected_mode: Expected telemetry mode.+ """+ telemetry_path = str(self.pipelex_dir / "telemetry.toml")+ verify_telemetry_config(telemetry_path, expected_mode)++ def verify_file_exists(self, relative_path: str) -> None:+ """Verify a file exists in the pipelex directory.++ Args:+ relative_path: Path relative to .pipelex directory.+ """+ file_path = self.pipelex_dir / relative_path+ assert file_path.exists(), f"Expected file {file_path} to exist"++ def verify_file_not_exists(self, relative_path: str) -> None:+ """Verify a file does not exist in the pipelex directory.++ Args:+ relative_path: Path relative to .pipelex directory.+ """+ file_path = self.pipelex_dir / relative_path+ assert not file_path.exists(), f"Expected file {file_path} to not exist"diff --git a/tests/integration/pipelex/cli/test_init_cmd.py b/tests/integration/pipelex/cli/test_init_cmd_backend_customization.pysimilarity index 68%rename from tests/integration/pipelex/cli/test_init_cmd.pyrename to tests/integration/pipelex/cli/test_init_cmd_backend_customization.pyindex 67433563c..089390851 100644--- a/tests/integration/pipelex/cli/test_init_cmd.py+++ b/tests/integration/pipelex/cli/test_init_cmd_backend_customization.py@@ -1,5 +1,3 @@-"""Integration tests for init command backend customization."""-from __future__ import annotationsimport shutil@@ -9,14 +7,33 @@ from typing import TYPE_CHECKINGif TYPE_CHECKING:from pytest_mock import MockerFixture-from pipelex.cli.commands.init_cmd import customize_backends_config, init_config-from pipelex.kit.paths import get_configs_dir+from pipelex.cli.commands.init.backends import customize_backends_config+from pipelex.cli.commands.init.config_files import init_config+from pipelex.kit.paths import get_kit_configs_dirfrom pipelex.tools.misc.toml_utils import load_toml_with_tomlkit-class TestBackendCustomization:- """Integration tests for backend customization during initialization."""+def get_backend_indices(backends_toml_path: str, backend_names: list[str]) -> list[int]:+ """Get 1-based indices for given backend names from backends.toml.++ Args:+ backends_toml_path: Path to backends.toml file.+ backend_names: List of backend keys to find indices for.++ Returns:+ List of 1-based indices corresponding to the backend names.+ """+ toml_doc = load_toml_with_tomlkit(backends_toml_path)+ backend_list: list[str] = [key for key in toml_doc if key != "internal"]+ indices: list[int] = []+ for name in backend_names:+ if name in backend_list:+ # 1-based index for user input+ indices.append(backend_list.index(name) + 1)+ return indices++class TestBackendCustomization:def test_customize_backends_config_with_default_selection(self, tmp_path: Path, mocker: MockerFixture) -> None:"""Test backend customization with default selection (pipelex_inference)."""# Setup directories with actual backends.toml@@ -24,18 +41,18 @@ class TestBackendCustomization:inference_dir.mkdir(parents=True)# Copy actual backends.toml from kit- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"test_backends = inference_dir / "backends.toml"shutil.copy2(actual_backends, test_backends)# Mock config_manager and user inputmock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)# Mock Console and Prompt to simulate user selecting default [1]- mocker.patch("pipelex.cli.commands.init_cmd.Console")- mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="1")+ mocker.patch("pipelex.cli.commands.init.backends.Console")+ mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Prompt.ask", return_value="1")# Executecustomize_backends_config()@@ -61,18 +78,23 @@ class TestBackendCustomization:inference_dir = tmp_path / ".pipelex" / "inference"inference_dir.mkdir(parents=True)- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"test_backends = inference_dir / "backends.toml"shutil.copy2(actual_backends, test_backends)+ # Dynamically get indices for the backends we want to test+ backend_names = ["openai", "anthropic", "mistral"]+ indices = get_backend_indices(str(actual_backends), backend_names)+ indices_str = ",".join(str(i) for i in indices)+# Mock config_managermock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)- # Mock user input: select [6,7,8] (openai, anthropic, mistral) in 1-based indexing- mocker.patch("pipelex.cli.commands.init_cmd.Console")- mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="6,7,8")+ # Mock user input with dynamically determined indices+ mocker.patch("pipelex.cli.commands.init.backends.Console")+ mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Prompt.ask", return_value=indices_str)# Executecustomize_backends_config()@@ -99,17 +121,22 @@ class TestBackendCustomization:inference_dir = tmp_path / ".pipelex" / "inference"inference_dir.mkdir(parents=True)- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"test_backends = inference_dir / "backends.toml"shutil.copy2(actual_backends, test_backends)+ # Dynamically get indices for the backends we want to test+ backend_names = ["pipelex_inference", "openai", "fal"]+ indices = get_backend_indices(str(actual_backends), backend_names)+ indices_str = " ".join(str(i) for i in indices)+mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(tmp_path / ".pipelex")- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)- # Mock user input with spaces: "1 6 12" in 1-based indexing- mocker.patch("pipelex.cli.commands.init_cmd.Console")- mocker.patch("pipelex.cli.commands.init_ui.Prompt.ask", return_value="1 6 12")+ # Mock user input with spaces and dynamically determined indices+ mocker.patch("pipelex.cli.commands.init.backends.Console")+ mocker.patch("pipelex.cli.commands.init.ui.backends_ui.Prompt.ask", return_value=indices_str)# Executecustomize_backends_config()@@ -136,7 +163,7 @@ class TestBackendCustomization:inference_dir = kit_configs_dir / "inference"inference_dir.mkdir()- actual_backends = Path(str(get_configs_dir())) / "inference" / "backends.toml"+ actual_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"shutil.copy2(actual_backends, inference_dir / "backends.toml")# Setup target directory@@ -144,10 +171,10 @@ class TestBackendCustomization:target_dir.mkdir()# Mock config_manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=str(kit_configs_dir))+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=str(kit_configs_dir))mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)mocker.patch("typer.echo")# Execute init_config@@ -176,9 +203,9 @@ class TestBackendCustomization:mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(config_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.backends.config_manager", mock_config_manager)- mock_console = mocker.patch("pipelex.cli.commands.init_cmd.Console")+ mock_console = mocker.patch("pipelex.cli.commands.init.backends.Console")# Execute - should not raise exceptioncustomize_backends_config()diff --git a/tests/integration/pipelex/cli/test_init_cmd_integration.py b/tests/integration/pipelex/cli/test_init_cmd_integration.pynew file mode 100644index 000000000..59a8ed7e2--- /dev/null+++ b/tests/integration/pipelex/cli/test_init_cmd_integration.py@@ -0,0 +1,301 @@+from __future__ import annotations++from pathlib import Path+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from pytest_mock import MockerFixture++from pipelex.cli.commands.init.command import init_cmd+from pipelex.cli.commands.init.ui.types import InitFocus+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path+from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper+++class TestInitCommandIntegration:+ def test_init_with_all_backends_selection(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test initialization with 'all' backends selection."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs: confirm init, select all backends, select telemetry OFF+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("all") # Select all backends+ env.add_prompt_input("1") # Telemetry: OFF++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify all backends are enabled (except internal which is separate)+ toml_doc = load_toml_with_tomlkit(str(env.inference_dir / "backends.toml"))+ for backend_key in toml_doc:+ if backend_key != "internal":+ assert toml_doc[backend_key]["enabled"] is True # type: ignore[index]++ # Verify routing was set to pipelex_first (since pipelex_inference is in "all")+ env.verify_routing("pipelex_first")++ # Verify telemetry+ env.verify_telemetry("off")++ def test_init_with_multiple_backends_custom_routing(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test initialization with multiple backends and custom routing."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for anthropic, mistral, openai (they will be selected in this order)+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "mistral", "openai"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select anthropic, mistral, openai+ env.add_prompt_input("1") # Primary backend: first one (anthropic)+ env.add_prompt_input("") # Accept default fallback order+ env.add_prompt_input("2") # Telemetry: ANONYMOUS++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify backends+ env.verify_backends_enabled(["openai", "anthropic", "mistral"])++ # Verify custom routing was created+ env.verify_routing("custom_routing", expected_default="anthropic")++ # Verify telemetry+ env.verify_telemetry("anonymous")++ def test_init_with_single_backend_auto_routing(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test initialization with single backend sets auto routing."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get index for openai+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["openai"])++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(str(indices[0])) # Select only openai+ env.add_confirm_input(True) # Confirm creating profile if needed+ env.add_prompt_input("3") # Telemetry: IDENTIFIED++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify only openai is enabled+ env.verify_backends_enabled(["openai"])++ # Verify routing is set to all_openai+ env.verify_routing("all_openai")++ # Verify telemetry+ env.verify_telemetry("identified")++ def test_init_config_only_focus(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test focused initialization: config files only."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # User inputs - CONFIG focus still runs full init on first time+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("1") # Backend selection (pipelex_inference)+ env.add_prompt_input("1") # Telemetry++ env.setup_mocks()++ # Execute with CONFIG focus+ init_cmd(focus=InitFocus.CONFIG, reset=False)++ # Verify config files exist+ env.verify_file_exists("pipelex.toml")+ env.verify_file_exists("inference/backends.toml")++ # CONFIG focus on first init triggers full flow+ env.verify_backends_enabled(["pipelex_inference"])+ env.verify_routing("pipelex_first")+ # Telemetry should be created+ if (env.pipelex_dir / "telemetry.toml").exists():+ env.verify_telemetry("off")++ def test_init_inference_only_focus(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test focused initialization: inference only."""+ # Setup environment with existing config+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)++ # Get indices for anthropic and mistral+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "mistral"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs - need to confirm reconfigure since backends already exist+ env.add_confirm_input(True) # Confirm reconfigure+ env.add_prompt_input(indices_str) # Select anthropic, mistral+ env.add_prompt_input("1") # Primary backend: first one (anthropic)+ env.add_prompt_input("") # Accept default fallback order++ env.setup_mocks()++ # Execute with INFERENCE focus+ init_cmd(focus=InitFocus.INFERENCE, reset=False)++ # Verify backends+ env.verify_backends_enabled(["anthropic", "mistral"])++ # Verify routing was configured (anthropic is first in selection)+ env.verify_routing("custom_routing", expected_default="anthropic")++ # Verify telemetry was NOT created (inference focus only)+ env.verify_file_not_exists("telemetry.toml")++ def test_init_telemetry_only_focus(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test focused initialization: telemetry only."""+ # Setup environment with existing config but no telemetry+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=False)++ # User inputs+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input("2") # Telemetry: ANONYMOUS++ env.setup_mocks()++ # Execute with TELEMETRY focus+ init_cmd(focus=InitFocus.TELEMETRY, reset=False)++ # Verify telemetry was created+ env.verify_telemetry("anonymous")++ def test_init_with_reset_flag(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test initialization with reset flag overwrites existing config."""+ # Setup environment with existing config+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # Modify telemetry to a different value+ telemetry_path = env.pipelex_dir / "telemetry.toml"+ toml_doc = load_toml_with_tomlkit(str(telemetry_path))+ toml_doc["telemetry_mode"] = "identified"+ save_toml_to_path(toml_doc, str(telemetry_path))++ # Get index for mistral+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["mistral"])++ # User inputs+ env.add_confirm_input(True) # Confirm reset+ env.add_prompt_input(str(indices[0])) # Select mistral only+ env.add_confirm_input(True) # Confirm creating profile if needed+ env.add_prompt_input("1") # Telemetry: OFF++ env.setup_mocks()++ # Execute with reset flag+ init_cmd(focus=InitFocus.ALL, reset=True)++ # Verify backends were reset+ env.verify_backends_enabled(["mistral"])++ # Verify telemetry was reset to OFF+ env.verify_telemetry("off")++ def test_init_reconfigure_existing_inference(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test reconfiguring inference when already configured."""+ # Setup environment with existing config+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)++ # Set pipelex_inference as enabled initially+ backends_path = env.inference_dir / "backends.toml"+ toml_doc = load_toml_with_tomlkit(str(backends_path))+ toml_doc["pipelex_inference"]["enabled"] = True # type: ignore[index]+ save_toml_to_path(toml_doc, str(backends_path))++ # Get index for openai+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["openai"])++ # User inputs+ env.add_confirm_input(True) # Confirm reconfigure+ env.add_prompt_input(str(indices[0])) # Change to openai+ env.add_confirm_input(True) # Confirm creating profile if needed++ env.setup_mocks()++ # Execute with INFERENCE focus+ init_cmd(focus=InitFocus.INFERENCE, reset=False)++ # Verify backend was changed+ env.verify_backends_enabled(["openai"])++ # Verify routing was updated+ env.verify_routing("all_openai")++ def test_init_two_backends_automatic_fallback(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test that two backends automatically set fallback without prompting order."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for exactly 2 backends (anthropic first, then openai)+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["anthropic", "openai"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs - note: no fallback order prompt for 2 backends+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select anthropic, openai+ env.add_prompt_input("1") # Primary backend: anthropic (first in selection)+ env.add_prompt_input("1") # Telemetry: OFF++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify backends+ env.verify_backends_enabled(["openai", "anthropic"])++ # Verify custom routing with automatic fallback order+ env.verify_routing("custom_routing", expected_default="anthropic", expected_fallback_order=["anthropic", "openai"])++ def test_init_pipelex_inference_sets_pipelex_first(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """Test that selecting pipelex_inference automatically sets pipelex_first routing."""+ # Setup environment+ env = MockedInitEnvironment(tmp_path, mocker)+ env.setup_empty_dir()++ # Get indices for pipelex_inference and openai+ kit_backends = Path(str(get_kit_configs_dir())) / "inference" / "backends.toml"+ indices = get_backend_indices_helper(str(kit_backends), ["pipelex_inference", "openai"])+ indices_str = ",".join(str(i) for i in indices)++ # User inputs - no primary/fallback prompts because pipelex_inference is included+ env.add_confirm_input(True) # Confirm initialization+ env.add_prompt_input(indices_str) # Select pipelex_inference and openai+ env.add_prompt_input("1") # Telemetry: OFF++ env.setup_mocks()++ # Execute+ init_cmd(focus=InitFocus.ALL, reset=False)++ # Verify backends+ env.verify_backends_enabled(["pipelex_inference", "openai"])++ # Verify routing is automatically set to pipelex_first+ env.verify_routing("pipelex_first")diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.pyindex 6653099a2..d82896fb8 100644--- a/tests/integration/pipelex/conftest.py+++ b/tests/integration/pipelex/conftest.py@@ -49,7 +49,8 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:# "gpt-5-chat",# "claude-4.5-sonnet",# "claude-4.1-opus",- "claude-4.5-sonnet",+ "claude-4.5-haiku",+ # "claude-4.5-sonnet",# "grok-3",# "grok-3-mini",# "base-claude",diff --git a/tests/integration/pipelex/plugins/conftest.py b/tests/integration/pipelex/plugins/conftest.pyindex 19a5312f2..9f8a8a13a 100644--- a/tests/integration/pipelex/plugins/conftest.py+++ b/tests/integration/pipelex/plugins/conftest.py@@ -1,5 +1,17 @@import pytest+from pipelex.cogt.exceptions import ModelManagerError+from pipelex.hub import get_models_manager+++def is_backend_available(backend_name: str) -> bool:+ """Check if an inference backend is available/enabled."""+ try:+ get_models_manager().get_required_inference_backend(backend_name)+ return True+ except ModelManagerError:+ return False+@pytest.fixture(params=["anthropic", "mistral", "meta", "amazon"])def bedrock_provider(request: pytest.FixtureRequest) -> str:diff --git a/tests/integration/pipelex/plugins/test_anthropic.py b/tests/integration/pipelex/plugins/test_anthropic.pyindex dc53af96a..0c03c4256 100644--- a/tests/integration/pipelex/plugins/test_anthropic.py+++ b/tests/integration/pipelex/plugins/test_anthropic.py@@ -9,6 +9,7 @@ from pipelex.plugins.anthropic.anthropic_exceptions import AnthropicSDKUnsupportfrom pipelex.plugins.anthropic.anthropic_llms import anthropic_list_available_modelsfrom pipelex.plugins.plugin_sdk_registry import Pluginfrom pipelex.system.environment import all_env_vars_are_set, any_env_var_is_placeholder+from tests.integration.pipelex.plugins.conftest import is_backend_availableREQUIRED_ENV_VARS = ["ANTHROPIC_API_KEY"]@@ -26,12 +27,14 @@ class TestAnthropic:pytestconfig: pytest.Config,plugin_for_anthropic: Plugin,):+ if not is_backend_available(plugin_for_anthropic.backend):+ pytest.skip(f"Backend '{plugin_for_anthropic.backend}' is not available or enabled")if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")if any_env_var_is_placeholder(REQUIRED_ENV_VARS):pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")try:- backend = get_models_manager().get_required_inference_backend("anthropic")+ backend = get_models_manager().get_required_inference_backend(plugin_for_anthropic.backend)anthropic_models_list = await anthropic_list_available_models(plugin=plugin_for_anthropic,backend=backend,diff --git a/tests/integration/pipelex/plugins/test_mistral.py b/tests/integration/pipelex/plugins/test_mistral.pyindex 23dca8dab..840a1a94b 100644--- a/tests/integration/pipelex/plugins/test_mistral.py+++ b/tests/integration/pipelex/plugins/test_mistral.py@@ -6,6 +6,7 @@ from rich.table import Tablefrom pipelex import pretty_printfrom pipelex.plugins.mistral.mistral_llms import mistral_list_available_modelsfrom pipelex.system.environment import all_env_vars_are_set, any_env_var_is_placeholder+from tests.integration.pipelex.plugins.conftest import is_backend_availableREQUIRED_ENV_VARS = ["MISTRAL_API_KEY"]@@ -17,6 +18,8 @@ class TestMistral:# pytest -k test_mistral_list_models -s -vv# make t VERBOSE=2 TEST=test_mistral_list_modelsdef test_mistral_list_models(self, pytestconfig: pytest.Config):+ if not is_backend_available("mistral"):+ pytest.skip("Backend 'mistral' is not available or enabled")if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")if any_env_var_is_placeholder(REQUIRED_ENV_VARS):@@ -49,6 +52,8 @@ class TestMistral:# pytest -k test_mistral_list_model_ids -s -vv# make t VERBOSE=2 TEST=test_mistral_list_model_idsdef test_mistral_list_model_ids(self, pytestconfig: pytest.Config):+ if not is_backend_available("mistral"):+ pytest.skip("Backend 'mistral' is not available or enabled")if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")if any_env_var_is_placeholder(REQUIRED_ENV_VARS):diff --git a/tests/integration/pipelex/plugins/test_openai.py b/tests/integration/pipelex/plugins/test_openai.pyindex fdd012a34..8e91ee774 100644--- a/tests/integration/pipelex/plugins/test_openai.py+++ b/tests/integration/pipelex/plugins/test_openai.py@@ -5,6 +5,7 @@ from pipelex.hub import get_models_managerfrom pipelex.plugins.openai.openai_llms import openai_list_available_modelsfrom pipelex.plugins.plugin_sdk_registry import Pluginfrom pipelex.system.environment import all_env_vars_are_set, any_env_var_is_placeholder+from tests.integration.pipelex.plugins.conftest import is_backend_available# make t VERBOSE=2 TEST=TestOpenAI@@ -18,6 +19,8 @@ class TestOpenAI:pytestconfig: pytest.Config,plugin_for_openai: Plugin,):+ if not is_backend_available(plugin_for_openai.backend):+ pytest.skip(f"Backend '{plugin_for_openai.backend}' is not available or enabled")match plugin_for_openai.backend:case "openai":required_env_vars = ["OPENAI_API_KEY"]diff --git a/tests/integration/pipelex/plugins/test_openai_image_gen.py b/tests/integration/pipelex/plugins/test_openai_image_gen.pyindex 7043b3c86..539c9a5d1 100644--- a/tests/integration/pipelex/plugins/test_openai_image_gen.py+++ b/tests/integration/pipelex/plugins/test_openai_image_gen.py@@ -7,6 +7,7 @@ from pipelex.plugins.plugin_sdk_registry import Pluginfrom pipelex.tools.misc.base_64_utils import save_base_64_str_to_binary_filefrom pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_pathfrom tests.conftest import TEST_OUTPUTS_DIR+from tests.integration.pipelex.plugins.conftest import is_backend_availablefrom tests.integration.pipelex.test_data import ImageGenTestCases@@ -16,6 +17,8 @@ from tests.integration.pipelex.test_data import ImageGenTestCasesclass TestImgGenByOpenAIGpt:@pytest.mark.parametrize(("topic", "image_desc"), ImageGenTestCases.IMAGE_DESC)async def test_gpt_image_generation(self, topic: str, image_desc: str):+ if not is_backend_available("openai"):+ pytest.skip("Backend 'openai' is not available or enabled")backend = get_models_manager().get_required_inference_backend("openai")client = OpenAIFactory.make_openai_client(Plugin(sdk="openai", backend="openai"),diff --git a/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx b/tests/test_pipelines/pipe_operators/pipe_llm_vision.plxindex a3e3645a8..0579c20e0 100644--- a/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx+++ b/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx@@ -25,7 +25,7 @@ type = "PipeLLM"description = "Describe what is in the image"inputs = { imageA = "Image", imageB = "Image" }output = "BasicDescription"-model = "llm_to_extract_diagram"+model = "llm_for_diagram_to_text"prompt = """Describe what you see in $imageA only.Completely ignore $imageB.@@ -36,7 +36,7 @@ type = "PipeLLM"description = "Describe what is in the image"inputs = { imageA = "Image", imageB = "Image" }output = "BasicDescription"-model = "llm_to_extract_diagram"+model = "llm_for_diagram_to_text"prompt = """Describe what you see in $imageB only.Completely ignore $imageA.@@ -48,7 +48,7 @@ type = "PipeLLM"description = "Provide detailed analysis of the image"inputs = { image = "Photo" }output = "VisionAnalysis"-model = "llm_to_extract_diagram"+model = "llm_for_diagram_to_text"system_prompt = "You are an expert image analyst. Provide detailed, accurate descriptions."prompt = """Analyze this image and describe what's the main topic etc.diff --git a/tests/test_pipelines/tricky_questions.plx b/tests/test_pipelines/tricky_questions.plxindex 70a0c931c..96a117249 100644--- a/tests/test_pipelines/tricky_questions.plx+++ b/tests/test_pipelines/tricky_questions.plx@@ -16,7 +16,7 @@ type = "PipeLLM"description = "Analyze a question to determine whether it's straightforward or tricky"inputs = { question = "Question" }output = "QuestionAnalysis"-model = "llm_for_complex_reasoning"+model = "llm_to_engineer"prompt = """Here is a question for an LLM:@question@@ -32,7 +32,7 @@ type = "PipeLLM"description = "Answer knowingly after analyzing a question"inputs = { question = "Question", question_analysis = "QuestionAnalysis" }output = "ThoughtfulAnswer"-model = "llm_for_complex_reasoning"+model = "llm_to_engineer"prompt = """A question was asked:@questiondiff --git a/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.py b/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.pyindex aa5f52ac9..e255b7ac2 100644--- a/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.py+++ b/tests/unit/builder/pipe/pipe_operator/pipe_llm/test_data.py@@ -54,7 +54,7 @@ class PipeLLMTestCases:inputs={"image": "Image"},output="Text",prompt="Analyze the image: $image",- llm_skill=LLMSkill.LLM_FOR_BASIC_VISION,+ llm_skill=LLMSkill.LLM_FOR_IMG_TO_TEXT_CHEAP,),PipeLLMBlueprint(source=None,@@ -63,7 +63,7 @@ class PipeLLMTestCases:inputs={"image": "Image"},output="Text",prompt="Analyze the image: $image",- model=LLMSkill.LLM_FOR_BASIC_VISION,+ model=LLMSkill.LLM_FOR_IMG_TO_TEXT_CHEAP,),)@@ -75,13 +75,13 @@ class PipeLLMTestCases:inputs={},output="Text",prompt="Generate text",- llm_skill=LLMSkill.LLM_TO_WRITE_QUESTIONS,+ llm_skill=LLMSkill.LLM_FOR_WRITING_CHEAP,),PipeLLMBlueprint(source=None,type="PipeLLM",description="Generate with preset",- model=LLMSkill.LLM_TO_WRITE_QUESTIONS,+ model=LLMSkill.LLM_FOR_WRITING_CHEAP,output="Text",prompt="Generate text",),@@ -95,13 +95,13 @@ class PipeLLMTestCases:inputs={},output="Text",prompt="Generate text",- llm_skill=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,+ llm_skill=LLMSkill.LLM_TO_ANSWER_QUESTIONS_CHEAP,),PipeLLMBlueprint(source=None,type="PipeLLM",description="Generate with settings",- model=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,+ model=LLMSkill.LLM_TO_ANSWER_QUESTIONS_CHEAP,output="Text",prompt="Generate text",),@@ -116,7 +116,7 @@ class PipeLLMTestCases:output="Analysis",system_prompt="You are a data analyst",prompt="Analyze: @data",- llm_skill=LLMSkill.LLM_TO_ANALYZE_DATA,+ llm_skill=LLMSkill.LLM_TO_ANSWER_QUESTIONS,),PipeLLMBlueprint(source=None,@@ -126,7 +126,7 @@ class PipeLLMTestCases:system_prompt="You are a data analyst",prompt="Analyze: @data",output="Analysis",- model=LLMSkill.LLM_TO_ANALYZE_DATA,+ model=LLMSkill.LLM_TO_ANSWER_QUESTIONS,),)@@ -138,7 +138,7 @@ class PipeLLMTestCases:inputs={},output="Item[]",prompt="Generate items",- llm_skill=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,+ llm_skill=LLMSkill.LLM_FOR_WRITING_CHEAP,),PipeLLMBlueprint(source=None,@@ -146,7 +146,7 @@ class PipeLLMTestCases:description="Generate multiple items",output="Item[]",prompt="Generate items",- model=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,+ model=LLMSkill.LLM_FOR_WRITING_CHEAP,),)@@ -158,7 +158,7 @@ class PipeLLMTestCases:inputs={},output="Item[5]",prompt="Generate items",- llm_skill=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,+ llm_skill=LLMSkill.LLM_FOR_WRITING_CHEAP,),PipeLLMBlueprint(source=None,@@ -166,7 +166,7 @@ class PipeLLMTestCases:description="Generate exactly 5 items",output="Item[5]",prompt="Generate items",- model=LLMSkill.LLM_TO_ANSWER_EASY_QUESTIONS,+ model=LLMSkill.LLM_FOR_WRITING_CHEAP,),)diff --git a/tests/unit/cli/test_init_cmd.py b/tests/unit/cli/test_init_cmd.pyindex 513a30594..a6d91dde6 100644--- a/tests/unit/cli/test_init_cmd.py+++ b/tests/unit/cli/test_init_cmd.py@@ -10,7 +10,7 @@ if TYPE_CHECKING:from pytest_mock import MockerFixture-from pipelex.cli.commands.init_cmd import init_config+from pipelex.cli.commands.init.config_files import init_configfrom pipelex.exceptions import PipelexCLIError@@ -25,11 +25,11 @@ class TestInitCmd:target_dir.mkdir()# Mock get_configs_dir and config manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)mock_echo = mocker.patch("typer.echo")# Execute@@ -51,11 +51,11 @@ class TestInitCmd:(target_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '0.9'")# Mock get_configs_dir and config manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)mock_echo = mocker.patch("typer.echo")# Execute@@ -84,11 +84,11 @@ class TestInitCmd:(target_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '0.9'")# Mock get_configs_dir and config manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)mock_echo = mocker.patch("typer.echo")# Execute@@ -121,11 +121,11 @@ class TestInitCmd:target_dir.mkdir()# Mock get_configs_dir and config manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)mock_echo = mocker.patch("typer.echo")# Execute@@ -152,11 +152,11 @@ class TestInitCmd:target_dir.mkdir()# Mock get_configs_dir and config manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)mocker.patch("shutil.copy2", side_effect=PermissionError("Permission denied"))# Execute and verify exception@@ -174,11 +174,11 @@ class TestInitCmd:target_dir = tmp_path / ".pipelex" # Don't create this# Mock get_configs_dir and config manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)mock_echo = mocker.patch("typer.echo")# Execute@@ -209,10 +209,10 @@ class TestInitCmd:target_dir.mkdir()# Mock config_manager- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)# Execute dry-runcount_dry = init_config(reset=False, dry_run=True)@@ -236,12 +236,12 @@ class TestInitCmd:target_dir.mkdir()# Mock config_manager and customize function- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)- mock_customize = mocker.patch("pipelex.cli.commands.init_cmd.customize_backends_config")+ mock_customize = mocker.patch("pipelex.cli.commands.init.backends.customize_backends_config")mocker.patch("typer.echo")# Execute@@ -261,12 +261,12 @@ class TestInitCmd:(target_dir / "pipelex.toml").write_text("[tool.pipelex]\nversion = '0.9'")# Mock config_manager and customize function- mocker.patch("pipelex.cli.commands.init_cmd.get_configs_dir", return_value=kit_configs_dir)+ mocker.patch("pipelex.cli.commands.init.config_files.get_kit_configs_dir", return_value=kit_configs_dir)mock_config_manager = mocker.MagicMock()mock_config_manager.pipelex_config_dir = str(target_dir)- mocker.patch("pipelex.cli.commands.init_cmd.config_manager", mock_config_manager)+ mocker.patch("pipelex.cli.commands.init.config_files.config_manager", mock_config_manager)- mock_customize = mocker.patch("pipelex.cli.commands.init_cmd.customize_backends_config")+ mock_customize = mocker.patch("pipelex.cli.commands.init.backends.customize_backends_config")mocker.patch("typer.echo")# Executediff --git a/tests/unit/cogt/model_routing/__init__.py b/tests/unit/cogt/model_routing/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/cogt/model_routing/test_routing_profile_fallback.py b/tests/unit/cogt/model_routing/test_routing_profile_fallback.pynew file mode 100644index 000000000..0ec9fb61c--- /dev/null+++ b/tests/unit/cogt/model_routing/test_routing_profile_fallback.py@@ -0,0 +1,272 @@+from pipelex.cogt.model_routing.routing_models import BackendMatchingMethod+from pipelex.cogt.model_routing.routing_profile import RoutingProfile+++class TestRoutingProfileFallbackOrder:+ """Tests for the fallback_order functionality in RoutingProfile."""++ def test_fallback_order_used_when_default_not_set(self):+ """Test that fallback_order[0] is used as primary backend when default is not set."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ fallback_order=["openai", "anthropic", "google"],+ )+ enabled_backends = ["openai", "anthropic", "google"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is not None+ assert result.backend_name == "openai"+ assert result.matching_method == BackendMatchingMethod.DEFAULT+ assert result.fallback_order == ["openai", "anthropic", "google"]++ def test_default_used_when_both_default_and_fallback_order_set(self):+ """Test that default is used as primary backend when both default and fallback_order are set."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ default="anthropic",+ fallback_order=["openai", "google"],+ )+ enabled_backends = ["openai", "anthropic", "google"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is not None+ assert result.backend_name == "anthropic"+ assert result.matching_method == BackendMatchingMethod.DEFAULT+ assert result.fallback_order == ["openai", "google"]++ def test_fallback_order_included_in_result_when_default_is_set(self):+ """Test that fallback_order is included in result even when default is used."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ default="openai",+ fallback_order=["anthropic", "google"],+ )+ enabled_backends = ["openai", "anthropic", "google"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is not None+ assert result.backend_name == "openai"+ assert result.fallback_order == ["anthropic", "google"]++ def test_nonexistent_backend_in_fallback_order_is_filtered_out(self):+ """Test that nonexistent backends in fallback_order are silently filtered out."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ fallback_order=["openai", "nonexistent_backend", "google"],+ )+ enabled_backends = ["openai", "google"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert - nonexistent_backend should be filtered out+ assert result is not None+ assert result.backend_name == "openai" # First enabled backend+ assert result.fallback_order == ["openai", "google"] # Only enabled backends++ def test_disabled_backend_in_fallback_order_is_filtered_out(self):+ """Test that disabled backends in fallback_order are silently filtered out."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ fallback_order=["openai", "anthropic", "google"],+ )+ enabled_backends = ["openai", "google"] # anthropic is disabled++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert - anthropic should be filtered out+ assert result is not None+ assert result.backend_name == "openai"+ assert result.fallback_order == ["openai", "google"] # Only enabled backends++ def test_current_behavior_preserved_when_neither_default_nor_fallback_order_set(self):+ """Test that None is returned when neither default nor fallback_order is set."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ )+ enabled_backends = ["openai", "anthropic", "google"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is None++ def test_exact_match_takes_precedence_over_fallback_order(self):+ """Test that exact route match takes precedence over fallback_order."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ routes={"test-model": "google"},+ fallback_order=["openai", "anthropic"],+ )+ enabled_backends = ["openai", "anthropic", "google"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is not None+ assert result.backend_name == "google"+ assert result.matching_method == BackendMatchingMethod.EXACT_MATCH+ assert result.fallback_order is None # Not DEFAULT match, so no fallback_order++ def test_pattern_match_takes_precedence_over_fallback_order(self):+ """Test that pattern match takes precedence over fallback_order."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ routes={"gpt-*": "openai"},+ fallback_order=["anthropic", "google"],+ )+ enabled_backends = ["openai", "anthropic", "google"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="gpt-4",+ )++ # Assert+ assert result is not None+ assert result.backend_name == "openai"+ assert result.matching_method == BackendMatchingMethod.PATTERN_MATCH+ assert result.fallback_order is None # Not DEFAULT match, so no fallback_order++ def test_fallback_order_with_single_backend(self):+ """Test that fallback_order works with a single backend."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ fallback_order=["openai"],+ )+ enabled_backends = ["openai", "anthropic"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is not None+ assert result.backend_name == "openai"+ assert result.fallback_order == ["openai"]++ def test_empty_fallback_order_treated_as_none(self):+ """Test that empty fallback_order is treated as if not set."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ fallback_order=[],+ )+ enabled_backends = ["openai", "anthropic"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is None++ def test_default_not_in_enabled_backends_falls_back_to_fallback_order(self):+ """Test that if default is not in enabled_backends, fallback_order is used."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ default="disabled_backend",+ fallback_order=["openai", "anthropic"],+ )+ enabled_backends = ["openai", "anthropic"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is not None+ assert result.backend_name == "openai" # First from fallback_order+ assert result.fallback_order == ["openai", "anthropic"]++ def test_fallback_order_filters_disabled_backends(self):+ """Test that disabled backends are filtered out from fallback_order."""+ # Arrange+ routing_profile = RoutingProfile(+ name="test_profile",+ fallback_order=["backend1", "backend2", "backend3"],+ )+ enabled_backends = ["backend1"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert - only enabled backends should remain+ assert result is not None+ assert result.backend_name == "backend1"+ assert result.fallback_order == ["backend1"] # backend2 and backend3 filtered out++ def test_fallback_order_preserved_in_result(self):+ """Test that the exact fallback_order list is preserved in the result."""+ # Arrange+ fallback_order = ["openai", "anthropic", "google", "mistral"]+ routing_profile = RoutingProfile(+ name="test_profile",+ default="openai",+ fallback_order=fallback_order,+ )+ enabled_backends = ["openai", "anthropic", "google", "mistral"]++ # Act+ result = routing_profile.get_backend_match_for_model(+ enabled_backends=enabled_backends,+ model_name="test-model",+ )++ # Assert+ assert result is not None+ assert result.fallback_order == fallback_order+ assert result.fallback_order is not fallback_order # Should be a copy/separate referencediff --git a/tests/unit/cogt/models/test_model_deck.py b/tests/unit/cogt/models/test_model_deck.pyindex 6b928f8e6..8b317663f 100644--- a/tests/unit/cogt/models/test_model_deck.py+++ b/tests/unit/cogt/models/test_model_deck.py@@ -1,10 +1,13 @@import pytest+from pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.exceptions import ModelWaterfallErrorfrom pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaultsfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.models.model_deck import ModelDeckfrom pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.runtime import ProblemReactionclass TestModelDeckGetOptionalInferenceModel:@@ -23,11 +26,14 @@ class TestModelDeckGetOptionalInferenceModel:def _create_test_model_deck(self,inference_models: dict[str, InferenceModelSpec] | None = None,- aliases: dict[str, str | list[str]] | None = None,+ aliases: dict[str, str] | None = None,+ fallbacks: dict[str, list[str]] | None = None,+ is_model_fallback_enabled: bool = False,) -> ModelDeck:return ModelDeck(inference_models=inference_models or {},aliases=aliases or {},+ waterfalls=fallbacks or {},llm_presets={},llm_choice_defaults=LLMSettingChoicesDefaults(for_text=LLMSetting(model="default_text", temperature=0.7, max_tokens=1000),@@ -37,6 +43,7 @@ class TestModelDeckGetOptionalInferenceModel:extract_choice_default="extract_text_from_visuals",img_gen_presets={},img_gen_choice_default="gen_image_basic",+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=is_model_fallback_enabled, missing_presets_reaction=ProblemReaction.NONE),)def test_direct_model_lookup_success(self):@@ -84,10 +91,14 @@ class TestModelDeckGetOptionalInferenceModel:def test_list_alias_resolution_first_success(self):# Arrangemodel_spec = self._create_test_model_spec("gpt-4")- model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": ["gpt-4", "claude-3"]})+ model_deck = self._create_test_model_deck(+ inference_models={"gpt-4": model_spec},+ fallbacks={"dummy-model-handle": ["gpt-4", "claude-3"]},+ is_model_fallback_enabled=True,+ )# Act- result = model_deck.get_optional_inference_model("best-model")+ result = model_deck.get_optional_inference_model("dummy-model-handle")# Assertassert result == model_spec@@ -97,32 +108,39 @@ class TestModelDeckGetOptionalInferenceModel:model_spec = self._create_test_model_spec("claude-3")model_deck = self._create_test_model_deck(inference_models={"claude-3": model_spec},- aliases={"best-model": ["nonexistent-model", "claude-3"]},+ fallbacks={"dummy-model-handle": ["nonexistent-model", "claude-3"]},+ is_model_fallback_enabled=True,)# Act- result = model_deck.get_optional_inference_model("best-model")+ result = model_deck.get_optional_inference_model("dummy-model-handle")# Assertassert result == model_specdef test_list_alias_resolution_none_found(self):# Arrange- model_deck = self._create_test_model_deck(aliases={"best-model": ["nonexistent-1", "nonexistent-2"]})-- # Act- result = model_deck.get_optional_inference_model("best-model")+ model_deck = self._create_test_model_deck(+ fallbacks={"dummy-model-handle": ["nonexistent-1", "nonexistent-2"]},+ is_model_fallback_enabled=True,+ )- # Assert- assert result is None+ # Act & Assert+ with pytest.raises(+ ModelWaterfallError,+ match=r"is a waterfall.*but none of the fallback models were found",+ ):+ model_deck.get_optional_inference_model("dummy-model-handle")def test_recursive_alias_resolution_success(self):# Arrangemodel_spec = self._create_test_model_spec("gpt-4")- model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": "best-gpt", "best-gpt": "gpt-4"})+ model_deck = self._create_test_model_deck(+ inference_models={"gpt-4": model_spec}, aliases={"dummy-model-handle": "best-gpt", "best-gpt": "gpt-4"}+ )# Act- result = model_deck.get_optional_inference_model("best-model")+ result = model_deck.get_optional_inference_model("dummy-model-handle")# Assertassert result == model_spec@@ -132,18 +150,23 @@ class TestModelDeckGetOptionalInferenceModel:model_spec = self._create_test_model_spec("gpt-4")model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec},- aliases={"best-model": ["nonexistent", "best-gpt"], "best-gpt": "gpt-4"},+ aliases={"best-gpt": "gpt-4"},+ fallbacks={"dummy-model-handle": ["nonexistent", "best-gpt"]},+ is_model_fallback_enabled=True,)# Act- result = model_deck.get_optional_inference_model("best-model")+ result = model_deck.get_optional_inference_model("dummy-model-handle")# Assertassert result == model_specdef test_empty_alias_list(self):# Arrange- model_deck = self._create_test_model_deck(aliases={"empty-alias": []})+ model_deck = self._create_test_model_deck(+ fallbacks={"empty-alias": []},+ is_model_fallback_enabled=True,+ )# Actresult = model_deck.get_optional_inference_model("empty-alias")@@ -172,11 +195,13 @@ class TestModelDeckGetOptionalInferenceModel:model_spec = self._create_test_model_spec("claude-3")model_deck = self._create_test_model_deck(inference_models={"claude-3": model_spec},- aliases={"best-model": ["premium-gpt", "premium-claude"], "premium-gpt": ["gpt-4-turbo", "gpt-4"], "premium-claude": "claude-3"},+ aliases={"premium-claude": "claude-3"},+ fallbacks={"dummy-model-handle": ["premium-gpt", "premium-claude"]},+ is_model_fallback_enabled=True,)# Act- result = model_deck.get_optional_inference_model("best-model")+ result = model_deck.get_optional_inference_model("dummy-model-handle")# Assertassert result == model_spec@@ -189,9 +214,12 @@ class TestModelDeckGetOptionalInferenceModel:inference_models={"gpt-4": model_spec1, "claude-3": model_spec2},aliases={"ai-model": "best-gpt", # string alias- "best-gpt": ["gpt-4-turbo", "gpt-4"], # list alias- "backup-model": ["claude-4", "claude-3"], # list alias},+ fallbacks={+ "best-gpt": ["gpt-4-turbo", "gpt-4"], # list fallback+ "backup-model": ["claude-4", "claude-3"], # list fallback+ },+ is_model_fallback_enabled=True,)# Actdiff --git a/tests/unit/kit/test_paths.py b/tests/unit/kit/test_paths.pyindex 4713cc79b..fe785ba1d 100644--- a/tests/unit/kit/test_paths.py+++ b/tests/unit/kit/test_paths.py@@ -1,4 +1,4 @@-from pipelex.kit.paths import get_agents_dir, get_configs_dir, get_kit_root+from pipelex.kit.paths import get_kit_agents_dir, get_kit_configs_dir, get_kit_rootclass TestKitPaths:@@ -12,12 +12,12 @@ class TestKitPaths:def test_get_agents_dir(self):"""Test that agents directory path is valid."""- agents_dir = get_agents_dir()+ agents_dir = get_kit_agents_dir()assert agents_dir is not Noneassert agents_dir.is_dir()def test_get_configs_dir(self):"""Test that configs directory path is valid."""- configs_dir = get_configs_dir()+ configs_dir = get_kit_configs_dir()assert configs_dir is not Noneassert configs_dir.is_dir()diff --git a/tests/unit/tools/config/test_load_config.py b/tests/unit/tools/config/test_load_config.pyindex d0c297161..68a7d79d5 100644--- a/tests/unit/tools/config/test_load_config.py+++ b/tests/unit/tools/config/test_load_config.py@@ -1,8 +1,17 @@from pipelex.config import PipelexConfig-from pipelex.hub import get_pipelex_hub+from pipelex.system.configuration.config_loader import config_manager+from pipelex.tools.misc.json_utils import deep_update+from pipelex.tools.misc.toml_utils import load_toml_from_pathclass TestLoadConfig:- def test_load_config(self):- hub = get_pipelex_hub()- hub.setup_config(config_cls=PipelexConfig, specific_config_path="pipelex/kit/configs/pipelex.toml")+ def test_load_kit_config(self):+ # The base condif must validate+ config = config_manager.load_config()+ PipelexConfig.model_validate(config)++ # The confiug updated by the kit config must validate too+ config_path = "pipelex/kit/configs/pipelex.toml"+ kit_config = load_toml_from_path(config_path)+ deep_update(config, kit_config)+ PipelexConfig.model_validate(config)diff --git a/tests/unit/tools/secrets/test_secrets_utils.py b/tests/unit/tools/secrets/test_secrets_utils.pyindex 1cbe8339e..6794b0c20 100644--- a/tests/unit/tools/secrets/test_secrets_utils.py+++ b/tests/unit/tools/secrets/test_secrets_utils.py@@ -16,44 +16,40 @@ def mock_secrets_provider(mocker: MockerFixture) -> Any:class TestSubstituteVars:- def test_simple_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_simple_secret_substitution(self, mock_secrets_provider: Any) -> None:mock_secrets_provider.get_secret.return_value = "secret_value"- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)- result = substitute_vars("Hello ${API_KEY}")+ result = substitute_vars("Hello ${API_KEY}", mock_secrets_provider)assert result == "Hello secret_value"mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")- def test_explicit_env_substitution(self, mocker: MockerFixture) -> None:+ def test_explicit_env_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:mocker.patch.dict(os.environ, {"TEST_VAR": "env_value"})- result = substitute_vars("Value: ${env:TEST_VAR}")+ result = substitute_vars("Value: ${env:TEST_VAR}", mock_secrets_provider)assert result == "Value: env_value"- def test_explicit_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_explicit_secret_substitution(self, mock_secrets_provider: Any) -> None:mock_secrets_provider.get_secret.return_value = "secret_value"- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)- result = substitute_vars("Value: ${secret:API_KEY}")+ result = substitute_vars("Value: ${secret:API_KEY}", mock_secrets_provider)assert result == "Value: secret_value"mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")- def test_fallback_env_to_secret(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_fallback_env_to_secret(self, mock_secrets_provider: Any) -> None:mock_secrets_provider.get_secret.return_value = "secret_fallback"- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)- result = substitute_vars("Value: ${env:MISSING_VAR|secret:FALLBACK_KEY}")+ result = substitute_vars("Value: ${env:MISSING_VAR|secret:FALLBACK_KEY}", mock_secrets_provider)assert result == "Value: secret_fallback"mock_secrets_provider.get_secret.assert_called_once_with(secret_id="FALLBACK_KEY")def test_env_takes_precedence_in_fallback(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:mocker.patch.dict(os.environ, {"EXISTING_VAR": "env_value"})- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)- result = substitute_vars("Value: ${env:EXISTING_VAR|secret:FALLBACK_KEY}")+ result = substitute_vars("Value: ${env:EXISTING_VAR|secret:FALLBACK_KEY}", mock_secrets_provider)assert result == "Value: env_value"# Secret should not be called since env var was found@@ -63,9 +59,8 @@ class TestSubstituteVars:"""Test fallback from secret to env when secret doesn't exist."""mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")mocker.patch.dict(os.environ, {"FALLBACK_ENV": "env_fallback"})- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)- result = substitute_vars("Value: ${secret:MISSING_SECRET|env:FALLBACK_ENV}")+ result = substitute_vars("Value: ${secret:MISSING_SECRET|env:FALLBACK_ENV}", mock_secrets_provider)assert result == "Value: env_fallback"mock_secrets_provider.get_secret.assert_called_once_with(secret_id="MISSING_SECRET")@@ -74,9 +69,8 @@ class TestSubstituteVars:"""Test that secret takes precedence when both secret and env are available in reverse order."""mock_secrets_provider.get_secret.return_value = "secret_value"mocker.patch.dict(os.environ, {"EXISTING_VAR": "env_value"})- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)- result = substitute_vars("Value: ${secret:EXISTING_SECRET|env:EXISTING_VAR}")+ result = substitute_vars("Value: ${secret:EXISTING_SECRET|env:EXISTING_VAR}", mock_secrets_provider)assert result == "Value: secret_value"mock_secrets_provider.get_secret.assert_called_once_with(secret_id="EXISTING_SECRET")@@ -85,85 +79,77 @@ class TestSubstituteVars:"""Test multiple variable substitutions in the same content."""mock_secrets_provider.get_secret.side_effect = lambda secret_id: f"secret_{secret_id}" # pyright: ignore[reportUnknownLambdaType]mocker.patch.dict(os.environ, {"ENV_VAR": "env_value"})- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)content = "API: ${API_KEY}, DB: ${env:ENV_VAR}, Token: ${secret:TOKEN}"- result = substitute_vars(content)+ result = substitute_vars(content, mock_secrets_provider)assert result == "API: secret_API_KEY, DB: env_value, Token: secret_TOKEN"- def test_no_placeholders(self) -> None:+ def test_no_placeholders(self, mock_secrets_provider: Any) -> None:"""Test content with no placeholders remains unchanged."""content = "This is just plain text with no variables"- result = substitute_vars(content)+ result = substitute_vars(content, mock_secrets_provider)assert result == content- def test_missing_env_var_raises_error(self) -> None:+ def test_missing_env_var_raises_error(self, mock_secrets_provider: Any) -> None:"""Test that missing required env var raises VarNotFoundError."""with pytest.raises(VarNotFoundError, match="Environment variable 'MISSING_VAR' is required but not set"):- substitute_vars("Value: ${env:MISSING_VAR}")+ substitute_vars("Value: ${env:MISSING_VAR}", mock_secrets_provider)- def test_missing_secret_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_missing_secret_raises_error(self, mock_secrets_provider: Any) -> None:"""Test that missing required secret raises error."""mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)with pytest.raises(VarNotFoundError, match="Could not get variable 'MISSING_SECRET': Secret not found"):- substitute_vars("Value: ${secret:MISSING_SECRET}")+ substitute_vars("Value: ${secret:MISSING_SECRET}", mock_secrets_provider)- def test_missing_default_secret_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_missing_default_secret_raises_error(self, mock_secrets_provider: Any) -> None:"""Test that missing default secret (no prefix) raises error."""mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)with pytest.raises(VarNotFoundError, match="Could not get variable 'MISSING_SECRET': Secret not found"):- substitute_vars("Value: ${MISSING_SECRET}")+ substitute_vars("Value: ${MISSING_SECRET}", mock_secrets_provider)- def test_fallback_both_missing_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_fallback_both_missing_raises_error(self, mock_secrets_provider: Any) -> None:mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)with pytest.raises(VarFallbackPatternError, match="Could not get variable from fallback pattern: env:MISSING_ENV\\|secret:MISSING_SECRET"):- substitute_vars("Value: ${env:MISSING_ENV|secret:MISSING_SECRET}")+ substitute_vars("Value: ${env:MISSING_ENV|secret:MISSING_SECRET}", mock_secrets_provider)- def test_reverse_fallback_both_missing_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_reverse_fallback_both_missing_raises_error(self, mock_secrets_provider: Any) -> None:"""Test that when both secret and env are missing in reverse order, error is raised."""mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)with pytest.raises(VarFallbackPatternError, match="Could not get variable from fallback pattern: secret:MISSING_SECRET\\|env:MISSING_ENV"):- substitute_vars("Value: ${secret:MISSING_SECRET|env:MISSING_ENV}")+ substitute_vars("Value: ${secret:MISSING_SECRET|env:MISSING_ENV}", mock_secrets_provider)def test_complex_variable_names(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:"""Test complex variable names with underscores, numbers, etc."""mock_secrets_provider.get_secret.return_value = "complex_secret"mocker.patch.dict(os.environ, {"API_KEY_V2": "complex_env"})- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)- result = substitute_vars("${env:API_KEY_V2} and ${DB_PASSWORD_123}")+ result = substitute_vars("${env:API_KEY_V2} and ${DB_PASSWORD_123}", mock_secrets_provider)assert result == "complex_env and complex_secret"- def test_whitespace_handling(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_whitespace_handling(self, mock_secrets_provider: Any) -> None:"""Test that whitespace around variable names is handled correctly."""mock_secrets_provider.get_secret.return_value = "secret_value"- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)# Note: whitespace should be preserved as part of the variable name- result = substitute_vars("Value: ${ API_KEY }")+ result = substitute_vars("Value: ${ API_KEY }", mock_secrets_provider)assert result == "Value: secret_value"mock_secrets_provider.get_secret.assert_called_once_with(secret_id=" API_KEY ")- def test_nested_braces_not_matched(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:+ def test_nested_braces_not_matched(self, mock_secrets_provider: Any) -> None:"""Test that nested braces are not matched by the pattern."""# The content has nested braces that should not be matched as a single variable# But ${nested} should be matched as a valid variablemock_secrets_provider.get_secret.return_value = "secret_nested"- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)content = "This ${has ${nested} braces} should not match"- result = substitute_vars(content)+ result = substitute_vars(content, mock_secrets_provider)# The ${nested} part should be substituted, but ${has ... should not be matched# because it contains a $ character which breaks the pattern@@ -174,34 +160,33 @@ class TestSubstituteVars:"""Test substitution in multiline content."""mock_secrets_provider.get_secret.return_value = "secret_value"mocker.patch.dict(os.environ, {"HOME": "/home/user"})- mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)content = """Line 1: ${API_KEY}Line 2: Some textLine 3: ${env:HOME}"""- result = substitute_vars(content)+ result = substitute_vars(content, mock_secrets_provider)expected = """Line 1: secret_valueLine 2: Some textLine 3: /home/user"""assert result == expected- def test_pattern_does_not_match_across_newlines(self) -> None:+ def test_pattern_does_not_match_across_newlines(self, mock_secrets_provider: Any) -> None:"""Test that the pattern doesn't match variables that span multiple lines."""content = """This ${VARNAME} should not match"""- result = substitute_vars(content)+ result = substitute_vars(content, mock_secrets_provider)# Should remain unchangedassert result == content- def test_pattern_does_not_match_with_quotes(self) -> None:+ def test_pattern_does_not_match_with_quotes(self, mock_secrets_provider: Any) -> None:"""Test that the pattern doesn't match variables containing quotes."""content = "This ${VAR\"NAME} and ${VAR'NAME} should not match"- result = substitute_vars(content)+ result = substitute_vars(content, mock_secrets_provider)# Should remain unchangedassert result == content- def test_unknown_prefix_raises_error(self) -> None:+ def test_unknown_prefix_raises_error(self, mock_secrets_provider: Any) -> None:"""Test that unknown prefix raises UnknownVarPrefixError."""with pytest.raises(UnknownVarPrefixError, match="Unknown variable prefix: 'foo'"):- substitute_vars("Value: ${foo:BAR}")+ substitute_vars("Value: ${foo:BAR}", mock_secrets_provider)diff --git a/tests/unit/tools/typing/test_custom_base_model.py b/tests/unit/tools/typing/test_custom_base_model.pynew file mode 100644index 000000000..e8f971855--- /dev/null+++ b/tests/unit/tools/typing/test_custom_base_model.py@@ -0,0 +1,25 @@+from pipelex.tools.misc.attribute_utils import AttributePolisher+from pipelex.tools.typing.pydantic_utils import CustomBaseModel+++class TestCustomBaseModel:+ """Tests for CustomBaseModel class."""++ def test_custom_base_model_truncates_repr(self) -> None:+ """Test that CustomBaseModel truncates long base64 and URL strings in repr output."""++ class TestModel(CustomBaseModel):+ base_64: str+ url: str+ other: str++ AttributePolisher.base_64_truncate_length = 10+ AttributePolisher.url_truncate_length = 10+ model = TestModel(+ base_64="b" * 20,+ url="data:image/png;base64," + "x" * 20,+ other="val",+ )+ repr_str = repr(model)+ assert "bbbbbbbbbb…" in repr_str+ assert "data:image…" in repr_strdiff --git a/tests/unit/tools/typing/test_enum_utilities.py b/tests/unit/tools/typing/test_enum_utilities.pynew file mode 100644index 000000000..73f0c0a01--- /dev/null+++ b/tests/unit/tools/typing/test_enum_utilities.py@@ -0,0 +1,47 @@+from pydantic import BaseModel, Field++from pipelex.tools.typing.pydantic_utils import (+ ExtraFieldAttribute,+ clean_model_to_dict,+ convert_strenum_to_str,+)+from pipelex.types import StrEnum+++class SimpleEnum(StrEnum):+ FIRST = "first"+ SECOND = "second"++ def display_name(self) -> str: # pragma: no cover - simple helper+ return self.value.upper()+++class ModelWithEnum(BaseModel):+ name: str+ enum_val: SimpleEnum+ secret: str = Field(+ "hidden",+ json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True},+ )+++class TestEnumUtilities:+ """Tests for enum utility functions."""++ def test_clean_model_to_dict_stringifies_enums_and_hides_fields(self) -> None:+ """Test that clean_model_to_dict converts enums to strings and hides marked fields."""+ model = ModelWithEnum(name="foo", enum_val=SimpleEnum.FIRST, secret="hidden")+ result = clean_model_to_dict(model)+ assert result == {"name": "foo", "enum_val": "FIRST"}++ def test_convert_strenum_to_str_recursive(self) -> None:+ """Test that convert_strenum_to_str recursively converts enums in nested structures."""+ data = {+ "enum": SimpleEnum.SECOND,+ "nested": [SimpleEnum.FIRST, {"e": SimpleEnum.SECOND}],+ }+ converted = convert_strenum_to_str(data)+ assert converted == {+ "enum": "SECOND",+ "nested": ["FIRST", {"e": "SECOND"}],+ }diff --git a/tests/unit/tools/typing/test_format_pydantic_validation_error.py b/tests/unit/tools/typing/test_format_pydantic_validation_error.pynew file mode 100644index 000000000..3548f733f--- /dev/null+++ b/tests/unit/tools/typing/test_format_pydantic_validation_error.py@@ -0,0 +1,98 @@+import pytest+from pydantic import BaseModel, ConfigDict, ValidationError++from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error+++class TestFormatPydanticValidationError:+ """Tests for format_pydantic_validation_error function."""++ def test_single_missing_and_extra_field(self) -> None:+ """Test formatting with a single missing and extra field."""++ class Validated(BaseModel):+ model_config = ConfigDict(extra="forbid")+ a: int+ b: str++ with pytest.raises(ValidationError) as exc:+ Validated.model_validate({"a": "not_int", "c": 1})++ formatted = format_pydantic_validation_error(exc.value)+ assert "Missing required fields: 'b'" in formatted+ assert "Extra forbidden fields: 'c'" in formatted++ def test_multiple_missing_fields(self) -> None:+ """Test formatting with multiple missing required fields displayed as comma-separated list."""++ class MultiRequired(BaseModel):+ model_config = ConfigDict(extra="forbid")+ field1: str+ field2: int+ field3: bool++ with pytest.raises(ValidationError) as exc:+ MultiRequired.model_validate({})++ formatted = format_pydantic_validation_error(exc.value)+ assert "Missing required fields:" in formatted+ assert "'field1'" in formatted+ assert "'field2'" in formatted+ assert "'field3'" in formatted+ # Check comma separation (order may vary)+ assert ", " in formatted++ def test_multiple_extra_fields(self) -> None:+ """Test formatting with multiple extra forbidden fields displayed as comma-separated list."""++ class NoExtras(BaseModel):+ model_config = ConfigDict(extra="forbid")+ valid_field: str++ with pytest.raises(ValidationError) as exc:+ NoExtras.model_validate({"valid_field": "ok", "extra1": 1, "extra2": 2, "extra3": 3})++ formatted = format_pydantic_validation_error(exc.value)+ assert "Extra forbidden fields:" in formatted+ assert "'extra1'" in formatted+ assert "'extra2'" in formatted+ assert "'extra3'" in formatted+ # Check comma separation+ assert ", " in formatted++ def test_combination_of_multiple_missing_and_extra_fields(self) -> None:+ """Test formatting with both multiple missing and extra fields together."""++ class MultiRequired(BaseModel):+ model_config = ConfigDict(extra="forbid")+ field1: str+ field2: int+ field3: bool++ with pytest.raises(ValidationError) as exc:+ MultiRequired.model_validate({"wrong1": "a", "wrong2": "b"})++ formatted = format_pydantic_validation_error(exc.value)+ assert "Missing required fields:" in formatted+ assert "Extra forbidden fields:" in formatted+ # Check for at least one missing field+ assert "'field1'" in formatted or "'field2'" in formatted or "'field3'" in formatted+ # Check for extra fields+ assert "'wrong1'" in formatted+ assert "'wrong2'" in formatted++ def test_model_type_errors(self) -> None:+ """Test formatting with model type validation errors."""++ class InnerModel(BaseModel):+ value: int++ class OuterModel(BaseModel):+ inner: InnerModel++ with pytest.raises(ValidationError) as exc:+ OuterModel.model_validate({"inner": "not_a_model"})++ formatted = format_pydantic_validation_error(exc.value)+ assert "Model type errors:" in formatted+ assert "expected InnerModel, got str" in formatteddiff --git a/tests/unit/tools/typing/test_pydantic_utils.py b/tests/unit/tools/typing/test_pydantic_utils.pydeleted file mode 100644index 26b010083..000000000--- a/tests/unit/tools/typing/test_pydantic_utils.py+++ /dev/null@@ -1,129 +0,0 @@-from typing import Any, cast--import pytest-from pydantic import BaseModel, ConfigDict, Field, ValidationError--from pipelex.tools.misc.attribute_utils import AttributePolisher-from pipelex.tools.typing.pydantic_utils import (- CustomBaseModel,- ExtraFieldAttribute,- clean_model_to_dict,- convert_strenum_to_str,- format_pydantic_validation_error,- serialize_model,-)-from pipelex.types import StrEnum---class ChildModel(BaseModel):- child_name: str- child_secret: str = Field(- ...,- json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True}, # This field should be hidden- )---class ParentModel(BaseModel):- parent_name: str- child: ChildModel---def test_serialize_model_with_hidden_fields():- # Create nested models- child = ChildModel(child_name="foo", child_secret="secret_foo")- parent = ParentModel(parent_name="bar", child=child)-- # Test default model_dump() shows everything- model_dump_result: dict[str, Any] = parent.model_dump()- child_dict = cast("dict[str, Any]", model_dump_result["child"])- assert "child_secret" in child_dict- assert child_dict["child_secret"] == "secret_foo"-- # Test our custom serializer omits hidden fields (child_secret)- serialized_result = cast("dict[str, Any]", serialize_model(parent))- child_serialized = cast("dict[str, Any]", serialized_result["child"])- assert "child_secret" not in child_serialized- assert serialized_result == {"parent_name": "bar", "child": {"child_name": "foo"}}---class SimpleEnum(StrEnum):- FIRST = "first"- SECOND = "second"-- def display_name(self) -> str: # pragma: no cover - simple helper- return self.value.upper()---class ModelWithEnum(BaseModel):- name: str- enum_val: SimpleEnum- secret: str = Field(- "hidden",- json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True},- )---def test_clean_model_to_dict_stringifies_enums_and_hides_fields() -> None:- model = ModelWithEnum(name="foo", enum_val=SimpleEnum.FIRST, secret="hidden")- result = clean_model_to_dict(model)- assert result == {"name": "foo", "enum_val": "FIRST"}---def test_convert_strenum_to_str_recursive() -> None:- data = {- "enum": SimpleEnum.SECOND,- "nested": [SimpleEnum.FIRST, {"e": SimpleEnum.SECOND}],- }- converted = convert_strenum_to_str(data)- assert converted == {- "enum": "SECOND",- "nested": ["FIRST", {"e": "SECOND"}],- }---def test_format_pydantic_validation_error() -> None:- class Validated(BaseModel):- model_config = ConfigDict(extra="forbid")- a: int- b: str-- with pytest.raises(ValidationError) as exc:- Validated.model_validate({"a": "not_int", "c": 1})-- formatted = format_pydantic_validation_error(exc.value)- assert "Missing required fields: ['b']" in formatted- assert "Extra forbidden fields: ['c: 1']" in formatted---def test_format_pydantic_validation_error_model_type() -> None:- class InnerModel(BaseModel):- value: int-- class OuterModel(BaseModel):- inner: InnerModel-- # Try to pass a dict where a specific model type is expected- with pytest.raises(ValidationError) as exc:- OuterModel.model_validate({"inner": "not_a_model"})-- formatted = format_pydantic_validation_error(exc.value)- assert "Model type errors:" in formatted- assert "expected InnerModel, got str" in formatted---def test_custom_base_model_truncates_repr() -> None:- class TestModel(CustomBaseModel):- base_64: str- url: str- other: str-- AttributePolisher.base_64_truncate_length = 10- AttributePolisher.url_truncate_length = 10- model = TestModel(- base_64="b" * 20,- url="data:image/png;base64," + "x" * 20,- other="val",- )- repr_str = repr(model)- assert "bbbbbbbbbb…" in repr_str- assert "data:image…" in repr_strdiff --git a/tests/unit/tools/typing/test_serialize_model.py b/tests/unit/tools/typing/test_serialize_model.pynew file mode 100644index 000000000..b8b1fbbf1--- /dev/null+++ b/tests/unit/tools/typing/test_serialize_model.py@@ -0,0 +1,40 @@+from typing import Any, cast++from pydantic import BaseModel, Field++from pipelex.tools.typing.pydantic_utils import ExtraFieldAttribute, serialize_model+++class ChildModel(BaseModel):+ child_name: str+ child_secret: str = Field(+ ...,+ json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True}, # This field should be hidden+ )+++class ParentModel(BaseModel):+ parent_name: str+ child: ChildModel+++class TestSerializeModel:+ """Tests for serialize_model function."""++ def test_serialize_model_with_hidden_fields(self) -> None:+ """Test that serialize_model omits fields marked as hidden while model_dump includes them."""+ # Create nested models+ child = ChildModel(child_name="foo", child_secret="secret_foo")+ parent = ParentModel(parent_name="bar", child=child)++ # Test default model_dump() shows everything+ model_dump_result: dict[str, Any] = parent.model_dump()+ child_dict = cast("dict[str, Any]", model_dump_result["child"])+ assert "child_secret" in child_dict+ assert child_dict["child_secret"] == "secret_foo"++ # Test our custom serializer omits hidden fields (child_secret)+ serialized_result = cast("dict[str, Any]", serialize_model(parent))+ child_serialized = cast("dict[str, Any]", serialized_result["child"])+ assert "child_secret" not in child_serialized+ assert serialized_result == {"parent_name": "bar", "child": {"child_name": "foo"}}